create millions of files for testing purposes

create_files.rb

#!/bin/env ruby

require 'thread/pool'
require 'ruby-progressbar'
require 'securerandom'
require 'etc' # to get the number of cores - http://stackoverflow.com/a/34215193
require 'colorize'
require 'sys/filesystem'
include Sys

$threads=Etc.nprocessors

# checks
if ARGV[0]==nil then abort "No directory given" end
unless File.directory?(ARGV[0]) then abort "#{ARGV[0]} is not a directory!" end
directory=ARGV[0]
$auto="FALSE"
if ARGV[1]=="AUTO" 
	puts "WARNING: You have selected \"AUTO\" mode. I'll keep creating files and subdirectories under #{directory.colorize(:yellow)} randomly until I run out of inodes or disk space."
	puts "WARNING: Type \"AUTO\" if you wish to proceed."
	answer=$stdin.gets.chomp
	unless answer=="AUTO" then abort "Exiting..".colorize(:red) end
	$auto="TRUE"
else

	if ARGV[1]==nil then abort "You need to give me the number of files" end
	if ARGV[2]==nil then abort "You need to give me the number of subdirectories" end


	# stolen from http://stackoverflow.com/a/1235891
	class String
	  def is_integer?
	    self.to_i.to_s == self
	  end
	end

	unless ARGV[1].is_integer? then abort "#{ARGV[1]} is not an integer!" end
	unless ARGV[2].is_integer? then abort "#{ARGV[2]} is not an integer!" end

	$number_of_files=ARGV[1].to_i
	$subdirs=ARGV[2].to_i
end


puts "INFO: Found #{$threads} cores, setting the threads to #{$threads}."
puts "INFO: Will use the directory: #{directory}."
if $auto=="FALSE"
	puts "INFO: Will create #{$subdirs} subdirectories."
	puts "INFO: Will create #{$number_of_files} per directory - this is a total of #{$number_of_files*$subdirs}."
	puts "QUESTION: If the above looks ok, type \"yes\"."
	answer=$stdin.gets.chomp
	unless answer=="yes" then abort "Exiting..".colorize(:red) end
end
puts "OK, changing directory to #{directory}.."
seconds=3
seconds.downto(0) do |n|
	puts "Starting in #{n} seconds..".colorize(:red)
	sleep 1
end

start_time = Time.now
def populate_directory(directory)

	check_space_and_inodes directory

	Dir.chdir(directory)

	new_dir=SecureRandom.urlsafe_base64(8)
	puts "Creating and switching to directory #{new_dir}.."
	Dir.mkdir(new_dir) unless File.exists?(new_dir)
	Dir.chdir(new_dir)

	filesizes_bytes=[1024,2048,4096,8192,16384,32768]

	pool = Thread.pool($threads)
        if $auto=="TRUE"
                $number_of_files=rand(1..100000)
                puts "INFO: will create #{$number_of_files} files"
        end
	progressbar=ProgressBar.create(:title => "Files", :starting_at => 0, :total => $number_of_files,
					:format => '%a |%B| %P%% (%R files per second) %c %t')

	$number_of_files.times {
	        pool.process {
			size=filesizes_bytes.sample
			filename=SecureRandom.urlsafe_base64(16)
			unless File.exists?(filename)
				size=(size*rand(0.0...3)).round # make the file size a little bigger
				content=SecureRandom.urlsafe_base64(size)	
				#puts "Creating file #{filename} with size #{size}.."
				File.open(filename, "w+") { |file| file.write(content) }
			else
				puts "File: #{filename} exists!!!!"
			end
		progressbar.increment
		}
	}
	
	pool.shutdown
	Dir.chdir("..")
end

def check_space_and_inodes(directory)
	mount=Filesystem.mount_point(directory)
	stats=Filesystem.stat(mount)
	if stats.percent_used > 95
		abort "Used over 95% of the filesystem #{mount}"
	else
		puts "Filesystem utilization for #{mount.colorize(:yellow)} at #{(stats.percent_used).round(2).to_s.colorize(:yellow)}%"
	end

	if stats.files_free < 200000
		abort "Less than 200000 inodes remaining!"
	else
                puts "Inodes remaining for #{mount.colorize(:yellow)} = #{stats.files_available.to_s.colorize(:yellow)}"   
	end
end

counter=0
if $auto=="FALSE"
	$subdirs.times {
		counter=counter+1
		populate_directory(directory)
		puts "\n----#{counter}/#{$subdirs}----".colorize(:blue)
	}
else
	loop do
		counter=counter+1
	        puts "\n----AUTO Mode: #{counter}----".colorize(:yellow)
		populate_directory(directory)	
	end
end

end_time = Time.now

total_time = end_time - start_time
puts "Total run took #{total_time} seconds".colorize(:yellow)