albrow · March 20, 2025 09:04 · Feb 23, 2013 · Dec 27, 2012 · Dec 27, 2012 · Dec 27, 2012
diff --git a/aws_deploy_tools.rb b/aws_deploy_tools.rb
@@ -13,11 +13,16 @@
 # Also includes the ability to invalidate cloudfront files
 # Used for deploying to s3/cloudfront
 class AWSDeployTools
-  
+	
 	def initialize(config = {})
 
 		@access_key_id = config['access_key_id']
 		@secret_access_key = config['secret_access_key']
+
+		# for privacy, allow user to store aws credentials in a shell env variable
+		@access_key_id ||= ENV['AWS_ACCESS_KEY_ID']
+		@secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY']
+
 		@bucket = config['bucket']
 		@acl = config['acl']
 		@cf_distribution_id = config['cf_distribution_id']
@@ -75,12 +80,33 @@ def push(s3_key, file, options = {})
 		if file.is_a? String
 			file = File.open(file, 'r')
 		end
+
+		# detect content type		
+		require 'mime/types'
+		file_path = file.path
+		# remove the .gz extension so the base extension will
+		# be used to determine content type. E.g. we want the
+		# type of index.html.gz to be text/html
+		if file_path.include? ".gz"
+			file_path.gsub!(".gz", "")
+		end
+		content_type = MIME::Types.type_for(File.extname(file_path)).first.to_s
+		content_type_hash = {:content_type => content_type}
+		options.merge! content_type_hash
 
 		puts "--> pushing #{file.path} to #{s3_key}...".green
     obj = @s3.buckets[@bucket].objects[s3_key]
     obj.write(file, options)
 
     @dirty_keys << s3_key
+    # Special cases for index files.
+    # for /index.html we should also invalidate /
+    # for /archive/index.html we should also invalidate /archive/
+    if (s3_key == "index.html")
+			@dirty_keys << "/"
+    elsif File.basename(s3_key) == "index.html"
+			@dirty_keys << s3_key.chomp(s3_key.split("/").last)
+    end
 
 	end
 
@@ -147,10 +173,19 @@ def invalidate(s3_keys)
 			return
 		elsif s3_keys.is_a? String
 			puts "--> invalidating #{s3_keys}...".yellow
-			paths = '<Path>/' + s3_keys + '</Path>'
+			# special case for root
+			if s3_keys == '/'
+				paths = '<Path>/</Path>'
+			else
+				paths = '<Path>/' + s3_keys + '</Path>'
+			end
 		elsif s3_keys.length > 0
 			puts "--> invalidating #{s3_keys.size} file(s)...".yellow
 			paths = '<Path>/' + s3_keys.join('</Path><Path>/') + '</Path>'
+			# special case for root
+			if s3_keys.include?('/')
+				paths.sub!('<Path>//</Path>', '<Path>/</Path>')
+			end
 		end
 
 		# digest calculation based on http://blog.confabulus.com/2011/05/13/cloudfront-invalidation-from-ruby/
@@ -181,7 +216,7 @@ def invalidate(s3_keys)
 		res = http.request(req)
 
 		if res.code == '201'
-			puts "Cloudfront Invalidation Success [201]".green
+			puts "Cloudfront Invalidation Success [201]. It may take a few minutes for the new files to propagate.".green
 		else
 			puts ("Cloudfront Invalidation Error: \n" + res.body).red
 		end
@@ -192,7 +227,6 @@ def invalidate(s3_keys)
 
 	# invalidates all the dirty keys and marks them as clean
 	def invalidate_dirty_keys
-
 		if @cf_distribution_id.nil?
 	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..."
 	    return

diff --git a/Rakefile b/Rakefile
@@ -115,3 +115,11 @@ task :compress_images do
 end
 
 # ...
+
+##
+# invoke system which to check if a command is supported
+# from http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
+# which('ruby') #=> /usr/bin/ruby
+def which(cmd)
+  system("which #{ cmd} > /dev/null 2>&1")
+end
diff --git a/aws_deploy_tools.rb b/aws_deploy_tools.rb
@@ -0,0 +1,207 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'openssl'
+require 'digest/sha1'
+require 'net/https'
+require 'base64'
+require 'aws-sdk'
+require 'digest/md5'
+require 'colored'
+
+
+# A convenient wrapper around aws-sdk
+# Also includes the ability to invalidate cloudfront files
+# Used for deploying to s3/cloudfront
+class AWSDeployTools
+
+	def initialize(config = {})
+
+		@access_key_id = config['access_key_id']
+		@secret_access_key = config['secret_access_key']
+		@bucket = config['bucket']
+		@acl = config['acl']
+		@cf_distribution_id = config['cf_distribution_id']
+		@dirty_keys = Set.new # a set of keys that are dirty (have been pushed but not invalidated)
+
+		if @bucket.nil?
+    	raise "ERROR: Must provide bucket name in constructor. (e.g. :bucket => 'bucket_name')" 
+	  end
+
+	  if @cf_distribution_id.nil?
+	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n Skipping cf invalidations..."
+	  end
+
+
+		@s3 = AWS::S3.new(config)
+
+	end
+
+	# checks if a local file is in sync with the s3 bucket
+	# file can either be a file object or a string with a
+	# valid file path
+	def synced?(s3_key, file)
+
+		if file.is_a? String
+			file = File.open(file, 'r')
+		end
+
+		f_content = file.read
+		obj_etag = ""
+
+		begin
+	    obj = @s3.buckets[@bucket].objects[s3_key]
+	    obj_etag = obj.etag # the etag is the md5 of the remote file
+	  rescue
+	   return false
+	  end
+
+		# the etag is surrounded by quotations. chomp removes them
+	  obj_etag = obj_etag.gsub('"', '')
+
+		# compare the etag to the md5 hash of the local file
+	  obj_etag == md5(f_content)
+
+	end
+
+	# pushes (writes) the file to the s3 bucket at location
+	# indicated by s3_key.
+	# file can either be a file object or a string with a
+	# valid file path
+	# options are any options that can be passed to the
+	# write method. 
+	# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
+	def push(s3_key, file, options = {})
+
+		if file.is_a? String
+			file = File.open(file, 'r')
+		end
+
+		puts "--> pushing #{file.path} to #{s3_key}...".green
+    obj = @s3.buckets[@bucket].objects[s3_key]
+    obj.write(file, options)
+
+    @dirty_keys << s3_key
+
+	end
+
+	# batch pushes (writes) the files to the s3 bucket at locations
+	# indicated by s3_keys. (more than one file at a time)
+	# files can either be a file object or a string with a
+	# valid file path
+	# options are any options that can be passed to the
+	# write method.
+	# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
+	def batch_push(s3_keys = [], files = [], options = {})
+
+		if (s3_keys.size != files.size) 
+			raise "ERROR: There must be a 1-to-1 correspondence of keys to files!"
+		end
+
+		files.each_with_index do |file, i|
+
+			s3_key = s3_keys[i]
+			push(s3_key, file, options)
+
+		end
+	end
+
+	# for each file, first checks if the file is synced.
+	# If not, it pushes (writes) the file.
+	# options are any options that can be passed to the
+	# write method.
+	# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
+	def sync(s3_keys = [], files = [], options = {})
+
+		if (s3_keys.size != files.size) 
+			raise "ERROR: There must be a 1-to-1 correspondence of keys to files!"
+		end
+
+		files.each_with_index do |file, i|
+
+			s3_key = s3_keys[i]
+	    unless synced?(s3_key, file)
+				push(s3_key, file, options)
+	    end
+
+		end
+
+	end
+
+
+	# a convenience method which simply returns the md5 hash of input
+	def md5 (input)
+	  Digest::MD5.hexdigest(input)
+	end
+
+	# invalidates files (accepts an array of keys or a single key)
+	# based heavily on https://gist.github.com/601408
+	def invalidate(s3_keys)
+
+		if @cf_distribution_id.nil?
+	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..."
+	    return
+	  end
+
+		if s3_keys.nil? || s3_keys.empty?
+			puts "nothing to invalidate."
+			return
+		elsif s3_keys.is_a? String
+			puts "--> invalidating #{s3_keys}...".yellow
+			paths = '<Path>/' + s3_keys + '</Path>'
+		elsif s3_keys.length > 0
+			puts "--> invalidating #{s3_keys.size} file(s)...".yellow
+			paths = '<Path>/' + s3_keys.join('</Path><Path>/') + '</Path>'
+		end
+
+		# digest calculation based on http://blog.confabulus.com/2011/05/13/cloudfront-invalidation-from-ruby/
+		date = Time.now.strftime("%a, %d %b %Y %H:%M:%S %Z")
+		digest = Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new('sha1'), @secret_access_key, date)).strip
+
+		uri = URI.parse('https://cloudfront.amazonaws.com/2010-08-01/distribution/' + @cf_distribution_id + '/invalidation')
+
+		if paths != nil
+		  req = Net::HTTP::Post.new(uri.path)
+		else
+		  req = Net::HTTP::Get.new(uri.path)
+		end
+
+		req.initialize_http_header({
+		  'x-amz-date' => date,
+		  'Content-Type' => 'text/xml',
+		  'Authorization' => "AWS %s:%s" % [@access_key_id, digest]
+		})
+
+		if paths != nil
+		  req.body = "<InvalidationBatch>" + paths + "<CallerReference>ref_#{Time.now.utc.to_i}</CallerReference></InvalidationBatch>"
+		end
+
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.use_ssl = true
+		http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+		res = http.request(req)
+
+		if res.code == '201'
+			puts "Cloudfront Invalidation Success [201]".green
+		else
+			puts ("Cloudfront Invalidation Error: \n" + res.body).red
+		end
+
+		return res.code
+
+	end
+
+	# invalidates all the dirty keys and marks them as clean
+	def invalidate_dirty_keys
+
+		if @cf_distribution_id.nil?
+	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..."
+	    return
+	  end
+
+		res_code = invalidate(@dirty_keys.to_a)
+		# mark the keys as clean iff the invalidation request went through
+		@dirty_keys.clear if res_code == '201'
+	end
+
+
+end
diff --git a/red_dragonfly.rb b/red_dragonfly.rb
@@ -0,0 +1,176 @@
+# A set of tools for minifying and compressing content
+# 
+# Currently supports:
+#   	- miniifying js, css, and html
+# 		- gzipping any content
+#     - compressing and shrinking images
+#
+# Currently depends on 3 system command line tools:
+#			- ImageMagick (http://www.imagemagick.org/script/index.php)
+# 		- gzip        (http://www.gzip.org/)
+#     - jitify      (http://www.jitify.com/)
+#
+# These may be swapped out for gem versions in the future,
+# but for now you have to manually install command line tools
+# above on your system if you don't already have them.
+#
+# Author: Alex Browne
+
+class RedDragonfly
+
+	def initialize
+		# ~~ gzip ~~
+		$gzip_options = {
+			:output_ext => "gz"
+		}
+
+		# ~~ minify (jitify) ~~
+		# $minify_options = {
+		#
+		# }
+
+
+		# ~~ images (ImageMagick) ~~
+		# exts          : files extensions which sould be minified during batch operations
+		# output_ext    : file extension of the output file ("" means keep the same extension)
+		# max_width     : max width for compressed images
+		# max_height    : max height for compressed images
+		# quality       : image compression quality (1-100, higher is better quality/bigger files)
+		# compress_type : type of compression to be used (http://www.imagemagick.org/script/command-line-options.php#compress)
+		$img_options = {
+			:output_ext => "jpg",
+			:max_width => "600",
+			:max_height => "1200",
+			:quality => "65",
+			:compress_type => "JPEG"
+		}
+	end
+
+	# accepts a single file or an array of files
+	# accepts a file object or the path to a file (a string)
+	# perserves the original file
+	# the output is (e.g.) .html.gz
+	def gzip (files = [])
+
+		unless which('gzip')
+			puts "WARNING: gzip is not installed on your system. Skipping gzip..."
+			return
+		end
+
+		unless files.is_a? Array
+			files = [files]
+		end
+
+		files.each do |file|
+
+			fname = get_filename(file)
+
+		  # invoke system gzip
+		  system("gzip -cn9 #{fname} > #{fname + '.' + $gzip_options[:output_ext]}")
+		end
+
+	end
+
+	# accepts a single file or an array of files
+	# accepts a file object or the path to a file
+	# overwrites the original file with the minified version
+	# html, css, and js supported only
+	def minify (files = [])
+
+		unless which('jitify')
+			puts "WARNING: jitify is not installed on your system. Skipping minification..."
+			return
+		end
+
+		unless files.is_a? Array
+			files = [files]
+		end
+
+		files.each do |file|
+
+			fname = get_filename(file)
+
+		  # invoke system jitify
+		  system("jitify --minify #{fname} > #{fname + '.min'}")
+		  # remove the .min extension
+		  system("mv #{fname + '.min'} #{fname}")
+
+		end
+
+	end
+
+	# compresses an image file using the options
+	# specified at the top
+	# accepts either a single file or an array of files
+	# accepts either a file object or a path to a file
+	def compress_img (files = [])
+
+		unless which('convert')
+			puts "WARNING: ImageMagick is not installed on your system. Skipping image compression..."
+			return
+		end
+
+		unless files.is_a? Array
+			files = [files]
+		end
+
+		files.each do |file|
+
+			fname = get_filename(file)
+
+			compress_cmd = "convert #{fname} -resize #{$img_options[:max_width]}x#{$img_options[:max_height]}\\>" + 
+		  					" -compress #{$img_options[:compress_type]} -quality #{$img_options[:quality]}" + 
+		  					" #{get_raw_filename(fname) + '.' + $img_options[:output_ext]}"
+
+		  # invoke system ImageMagick
+		  system(compress_cmd)
+		  # remove the old file (if applicable)
+		  if (get_ext(fname) != ("." + $img_options[:output_ext]))
+		  	system("rm #{fname}")
+		  end
+
+		end
+
+	end
+
+	# returns the filename (including path and ext) if the input is a file
+	# if the input is a string, returns the same string
+	def get_filename (file)
+		if file.is_a? File
+			file = file.path
+		end
+		return file
+	end
+
+	# returns the extension of a file
+	# accepts either a file object or a string path
+	def get_ext (file)
+
+		if file.is_a? String
+			return File.extname(file)
+		elsif file.is_a? File
+			return File.extname(file.path)
+		end
+
+	end
+
+	# returns the raw filename (minus extension) of a file
+	# accepts either a file object or a string path
+	def get_raw_filename (file)
+
+		# convert to string
+		file = get_filename(file)
+		# remove extension
+		file.sub(get_ext(file), "")
+
+	end
+
+	##
+	# invoke system which to check if a command is supported
+	# from http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
+	# which('ruby') #=> /usr/bin/ruby
+	def which(cmd)
+	  system("which #{ cmd} > /dev/null 2>&1")
+	end
+
+end
diff --git a/Rakefile (an excerpt) → Rakefile b/Rakefile (an excerpt) → Rakefile
@@ -1,3 +1,5 @@
+# ...
+
 desc "Deploy website to s3/cloudfront via aws-sdk"
 task :s3_cloudfront => [:generate, :minify, :gzip, :compress_images] do
   puts "=================================================="
@@ -111,3 +113,5 @@ task :compress_images do
   puts "DONE."
 
 end
+
+# ...
diff --git a/Rakefile (an excerpt) b/Rakefile (an excerpt)
@@ -110,4 +110,4 @@ task :compress_images do
 
   puts "DONE."
 
-end
+end
diff --git a/Rakefile (an excerpt) b/Rakefile (an excerpt)
@@ -0,0 +1,113 @@
+desc "Deploy website to s3/cloudfront via aws-sdk"
+task :s3_cloudfront => [:generate, :minify, :gzip, :compress_images] do
+  puts "=================================================="
+  puts "      Deploying to Amazon S3 & CloudFront"
+  puts "=================================================="
+
+  # setup the aws_deploy_tools object
+  config = YAML::load( File.open("_config.yml"))
+  aws_deploy = AWSDeployTools.new(config)
+
+  # get all files in the public directory
+  all_files = Dir.glob("#{$public_dir}/**/*.*")
+
+  # we want the gzipped version of the files, not the regular (non-gzipped) version
+  # excluded files contains all the regular versions, which will not be deployed
+  excluded_files = []
+  $gzip_exts.collect do |ext|
+    excluded_files += Dir.glob("#{$public_dir}/**/*.#{ext}")
+  end
+
+  # we do gzipped files seperately since they have different metadata (:content_encoding => gzip)
+  puts "--> syncing gzipped files...".yellow
+  gzipped_files = Dir.glob("#{$public_dir}/**/*.gz")
+  gzipped_keys = gzipped_files.collect {|f| (f.split("#{$public_dir}/")[1]).sub(".gz", "")}
+
+  aws_deploy.sync(gzipped_keys, gzipped_files,
+          :reduced_redundancy => true,
+          :cache_control => "max_age=86400", #24 hours
+          :content_encoding => 'gzip',
+          :acl => config['acl']
+          )
+
+  puts "--> syncing all other files...".yellow
+  non_gzipped_files = all_files - gzipped_files - excluded_files
+  non_gzipped_keys = non_gzipped_files.collect {|f| f.split("#{$public_dir}/")[1]}
+
+  aws_deploy.sync(non_gzipped_keys, non_gzipped_files,
+          :reduced_redundancy => true,
+          :cache_control => "max_age=86400", #24 hours
+          :acl => config['acl']
+          )
+
+  # invalidate all the files we just pushed
+  aws_deploy.invalidate_dirty_keys  
+
+  puts "DONE."
+
+end
+
+desc "Compress all applicable content in public/ using gzip"
+task :gzip do
+
+  unless which('gzip')
+    puts "WARNING: gzip is not installed on your system. Skipping gzip..."
+    return
+  end
+
+  @compressor ||= RedDragonfly.new
+
+  $gzip_exts.each do |ext|
+    puts "--> gzipping all #{ext}...".yellow
+    files = Dir.glob("#{$gzip_dir}/**/*.#{ext}")
+    files.each do |f|
+      @compressor.gzip(f)
+    end
+  end
+
+  puts "DONE."
+end
+
+desc "Minify all applicable files in public/ using jitify"
+task :minify do
+
+  unless which('jitify')
+    puts "WARNING: jitify is not installed on your system. Skipping minification..."
+    return
+  end
+
+  @compressor ||= RedDragonfly.new
+
+  $minify_exts.each do |ext|
+    puts "--> minifying all #{ext}...".yellow
+    files = Dir.glob("#{$minify_dir}/**/*.#{ext}")
+    files.each do |f|
+      @compressor.minify(f)
+    end
+  end
+
+  puts "DONE."
+
+end
+
+desc "Compress all images in public/ using ImageMagick"
+task :compress_images do
+
+  unless which('convert')
+    puts "WARNING: ImageMagick is not installed on your system. Skipping image compression..."
+    return
+  end
+
+  @compressor ||= RedDragonfly.new
+
+  $compress_img_exts.each do |ext|
+    puts "--> compressing all #{ext}...".yellow
+    files = Dir.glob("#{$compress_img_dir}/**/*.#{ext}")
+    files.each do |f|
+      @compressor.compress_img(f)
+    end
+  end
+
+  puts "DONE."
+
+end