Some very useful utilities

ReferenceFile.sh

#To start lightweight python http server
python -m SimpleHTTPServer 8000

#To listen on tcp port for incoming messages
sudo tcpflow -i any -C -J port 1234

find . -mtime -4  # find files modified less than 4 days ago
find . -mtime 1   # find files modified between 24 and 48 hours ago
find . -mtime +4  # find files modified more than 4 days ago

#Delete all files in a directory:
find /path/to/files* -type f -delete

#Delete all files in a directory older than 5 days:
find /path/to/files* -mtime +5 -type f -delete

find * -mtime +90 -print0 | xargs -r0 rm -rf #To handle file names with spaces etc.
find /path/to/search -name "*.doc" -exec cp {} /path/to/copy/to \;

#To check who is logged in using SFTP
ps -ef | grep '[s]shd:.*@notty' | grep -v ^root

#Find Non-ascii characters
grep --color='auto' -P -n "[\x80-\xFF]" file.xml
perl -nE'say$.if/[\xE0-\xFF]/'
VIM HILIGHT NON-ASCII:
/[^\x00-\x7F]

## Django-South Basics ##
#after creating the model first time
python manage.py syncdb --migrate
python manage.py schemamigration leads --initial

#to apply the migration first time, first do syncdb then do:
python manage.py migrate leads

#if you have done syncdb, or want to skip a migration
python manage.py migrate leads 0001 --fake

#To create migration after changing model
./manage.py schemamigration leads --auto

#To apply un-applied migrations to db
python manage.py migrate leads

## Amazon EC2, Commands to Remember ##
sudo vim /etc/apparmor.d/usr.sbin.mysqld
sudo invoke-rc.d apparmor restart

#After attaching EBS volume (which will get mounted as /dev/xvdh for /dev/sdh)
sudo apt-get install -y xfsprogs
grep -q xfs /proc/filesystems || sudo modprobe xfs
sudo mkfs.xfs /dev/sdh
echo "/dev/sdh /vol xfs noatime 0 0" | sudo tee -a /etc/fstab
sudo mkdir -m 000 /vol
sudo mount /vol

## OR ##
sudo apt-get install -y xfsprogs
sudo mkfs.xfs /dev/xvdd
sudo mkdir volume_name
sudo mount /dev/xvdd volume_name

## Unix Redirection ##
>       #Redirect standard output
2>      #Redirect standard error
2>&1    #Redirect standard error to standard output
<       #Redirect standard input
|       #Pipe standard output to another command
>>      #Append to standard output
2>&1|   #Pipe standard output and standard error to another command


## audio awk stuff
cat /dev/urandom | fold -b1 | awk '{for (i=0;i<100;i++) printf($1);}' > /dev/dsp

############ MYSQL ##############
GRANT SELECT, INSERT, DELETE, UPDATE ON dbname.* TO 'backend'@'66.160.141.87' IDENTIFIED BY 'yourpass';

sudo mysqladmin --defaults-file=/usr/local/mysql/share/mysql/my-medium.cnf shutdown to shut mysqlserver
sudo /etc/init.d/mysql    to start mysql
mysql --socket=/tmp/mysql.socket -uroot -pbadams

#If you have changed your datadir location, do this, else it will give problems
sudo vim /etc/apparmor.d/usr.sbin.mysqld
sudo invoke-rc.d apparmor restart
sudo mysql_install_db

select c.source, DATE(c.created_on), c.status, COUNT(*) INTO OUTFILE '/tmp/cwc.csv' FIELDS TERMINATED BY '||' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n' from cora_entry c group by c.source ,DATE(c.created_on), c.status;

sudo mkdir /var/run/mysqld; sudo ln -s /tmp/mysql.socket /var/run/mysqld/mysqld.sock

#openDNS:
208.67.222.222
208.67.220.220

#Solr-Highlighting
http://localhost:8080/solr/select/?q=video&version=2.2&start=0&rows=10&indent=on&hl=true&hl.simple.pre=%3Cstrong%3E&hl.simple.post=%3C/strong%3E&hl.fl=*

#Solr-Tags
http://www.balajidigital.com:8080/ClassifiedsSolr/select/?q=ndtv&facet=true&facet.field=keywords&facet.field=description

#Port Forwarding:
ssh -L 8000:localhost:8000 [email protected]

###Reverse SSH tunneling a shell###

#(step 1, from the computer you wish to access)
derwiki@firewalledcomputer:~$ ssh -R localhost:2002:localhost:22 mypublicserver.com

#(step 2, from any computer than can access mypublicserver.com)
derwiki@mylaptopontheinternet:~$ ssh mypublicserver.com -p 2002

# Dump msql
mysqldump -uroot -pasd -c icicipru --default-character-set=utf8  --no-create-db --no-create-info > /tmp/t.sql

###mysql unicode issue##

put "default-character-set=utf8" below [mysqld] (in my.cnf) and restart mysql
restart mysql.... and recreate the table... or run $> alter database charset utf8 and ALTER TABLE <table_name> charset=utf8;
show create <dbname>;    at mysql prompt to make sure that default is not latin.... but utf-8

### if mysqldump looses encoding ###
mysqldump -uroot -pasd  --default-character-set=latin1 qbank > OLD_latin1-utf8.sql
replace "CHARSET=latin1" "CHARSET=utf8" "CHARSET=latin" "CHARSET=utf8" "SET NAMES latin1" "SET NAMES utf8" < OLD_latin1-utf8.sql > NEW_UTF8_PROPER_Sep7.sql
mysql -uroot -pasd qbank < NEW_UTF8_PROPER_DATA.sql
The key is to make sure that client application also encodes it only once. you may have to change my.cnf to make client AND server latin1, then put dump back, then get a pure latin1 dump, then convert it into a pure utf8 dump, then again change my.cnf for both client and server to utf8 and dump back

#Ignore files in Subversion?
svn propedit svn:ignore .

##RANDOM HACKS##
----------------------------------
#change access, modified and change times for file after editting
touch -at 200010161000 <filename>  #changes access time for file to 2000-10-16, -mt, -ct for modified and change times
#remove history for last command
rm -f /var/log/wtmp && touch /var/log/wtmp
>/var/log/lastlog  # puts blank in lastlog
#ssh logs in auth.log, remove these to remove traces of ssh history
cat /var/log/auth.log | grep sshd
----------------------------------

vim increment all "x" between markers from i=1 to ..
put markers using ma,mb ... jump to markers using 'a ,  'b
:let i=1 | 'a,'bg/x/s//\=i/g |let i=i+1

#To link several directories into a subdirectory: To be run from current directory
   ln -s $HOME/hyperhelp $HOME/metacard experimental

-----------------------------------------------
from IPython.frontend.terminal.embed import InteractiveShellEmbed
InteractiveShellEmbed()()

##Global GIT setup:

##Configure git
git config --global user.name "Your Name"
git config --global user.email "[email protected]"

##Existing Git Repo?

## adding remote origin in existing git dir ##
## cd existing_git_repo
git remote add origin [email protected]:yourusername/yourreponame.git

##push commits to remote master
git push origin master

##Single command for add and commit
git commit -a

## push to current branch on remote
git push

## Powerful git GUI
gitk --all

##To get a git project on local machine:
git clone git://github.com/git/hello-world.git

#Git local commit diff with development
git diff origin/master..HEAD

#Git revert local commit
git reset HEAD^

#This will undo the last commit and push the updated history to the remote. You need to pass the -f because you're replacing upstream history in the remote.
git reset --hard HEAD~1
git push -f

#This will cancel ALL local commits and make repo equal to local
git reset --hard origin/master

#To see list of files which differ between two branches
git diff --name-status master..mark

#Adding network latency: bandwidth, slow
tc qdisc add dev lo root handle 1:0 netem delay 100msec
#restore to normal:
tc qdisc del dev lo root

#Get rid of everything local and make my local equal to remote
git fetch origin 
git reset --hard origin/master


## ufw ##
-----------------------------
#allow port 22 for tcp/udp
sudo ufw allow 22
# delete the rule
sudo ufw delete deny 22
# start firewall rules
sudo ufw enable
# stop firewall rules
sudo ufw disable
# get status of rules
sudo ufw status
# allow ip range for port 10080
sudo ufw allow from 192.168.1.0/24 to any port 10080
#block specific IP
sudo ufw deny from 207.46.232.182
#rules files
sudo vim /var/lib/ufw/user.rules

# by default all is blocked, so before enabling, open ssh port
# for intranet only, open IP range for a specific port
# Whichever rule is first encountered will be executed, following rules will be ignored
-----------------------------

mencoder inputfile.VOB -of lavf -ovc lavc -lavcopts vcodec=flv:vbitrate=150 -ofps 25 -oac mp3lame -lameopts abr:br=32 -srate 22050 -vf scale=720 -o outputfile.flv
ffmpeg -i VTS_01_1.VOB -f flv -vcodec flv -b 200k -r 25 -s 296x236 -ab 64 -ar 44100 -ac 2 -acodec libmp3lame test8.flv
ffmpeg -i VTS_01_1.VOB -f flv -vcodec flv -b 200k -r 25 -s 296x236 -ab 64 -ar 44100 -ac 2 -acodec mp3 test8.flv

##Screen##
--------
screen -ls   #list all screens
screen -r <screen_name>  #to attach to a screen which is detached currently
screen -rx <screen_name>  #to force attach a screen which is attached to someone
Ctrl-A "  #to see list of screens inside a screen
Ctrl-A D  #to logout of screen without closing it
--------

#remove spaces from file name ------
find . -name "* *"|while read file
do
    i=`echo "$file"| nawk ' BEGIN {OFS="-"} $1=$1 '`
    echo "mv \"$file\" $i"
done
--------------------------------

#javascript editable page hack
javascript:document.body.contentEditable='true'; document.designMode='on'; void 0
<a href="javascript:void(0);" class="open_upload">+ upload photos</a>
--------------------------------

#Rsync and ssh resume partial loaded files
rsync --partial --progress --rsh=ssh local_file user@host:remote_file

python -m smtpd -n -c DebuggingServer localhost:25

sed -i 's/ugly/beautiful/g' /home/bruno/old-friends/sue.txt

##############################################################################
###Find and LS##
  ls /usr/bin | pr -T9 -W$COLUMNS	#Print in 9 columns to width of terminal
 	find -name '*.[ch]' | xargs grep -E 'expr'	#Search 'expr' in this dir and below. See also findrepo
 	find -type f -print0 | xargs -r0 grep -F 'example'	#Search all regular files for 'example' in this dir and below
 	find -maxdepth 1 -type f | xargs grep -F 'example'	#Search all regular files for 'example' in this dir
 	find -maxdepth 1 -type d | while read dir; do echo $dir; echo cmd2; done	#Process each item with multiple commands (in while loop)
	find -type f ! -perm -444	#Find files not readable by all (useful for web site)
	find -type d ! -perm -111	#Find dirs not accessible by all (useful for web site)
	locate -r 'file[^/]*\.txt'	#Search cached index for names. This re is like glob *file*.txt
	#look reference	Quickly search (sorted) dictionary for prefix
	grep --color reference /usr/share/dict/words	#Highlight occurances of regular expression in dictionary

##############################################################################

## Compression And Archives ##
  gpg -c file	Encrypt file
 	gpg file.gpg	Decrypt file
 	tar -c dir/ | bzip2 > dir.tar.bz2	#Make compressed archive of dir/
 	bzip2 -dc dir.tar.bz2 | tar -x	#Extract archive (use gzip instead of bzip2 for tar.gz files)
 	tar -c dir/ | gzip | gpg -c | ssh user@remote 'dd of=dir.tar.gz.gpg'	#Make encrypted archive of dir/ on remote machine
 	find dir/ -name '*.txt' | tar -c --files-from=- | bzip2 > dir_txt.tar.bz2	#Make archive of subset of dir/ and below
 	find dir/ -name '*.txt' | xargs cp -a --target-directory=dir_txt/ --parents	#Make copy of subset of dir/ and below
 	( tar -c /dir/to/copy ) | ( cd /where/to/ && tar -x -p )	#Copy (with permissions) copy/ dir to /where/to/ dir
 	( cd /dir/to/copy && tar -c . ) | ( cd /where/to/ && tar -x -p )	#Copy (with permissions) contents of copy/ dir to /where/to/
 	( tar -c /dir/to/copy ) | ssh -C user@remote 'cd /where/to/ && tar -x -p' 	#Copy (with permissions) copy/ dir to remote:/where/to/ dir
 	dd bs=1M if=/dev/sda | gzip | ssh user@remote 'dd of=sda.gz'	#Backup harddisk to remote machine
###############################################################################



##############################################################################

#Like top, but for files
watch -d -n 2 ‘df; ls -FlAt;’

#Download an entire website
wget –random-wait -r -p -e robots=off -U mozilla http://www.example.com
## -p parameter tells wget to include all files, including images.
## -e robots=off you don’t want wget to obey by the robots.txt file
## -U mozilla as your browsers identity.
## –random-wait to let wget chose a random number of seconds to wait, avoid get into black list.
### Other Useful wget Parameters ###
## –limit-rate=20k limits the rate at which it downloads files.
## -b continues wget after logging out.
## -o $HOME/wget_log.txt logs the output

#List the size (in human readable form) of all sub folders from the current location
du -h –max-depth=1

#A very simple and useful stopwatch
time read (ctrl-d to stop)
time read -sn1 (s:silent, n:number of characters. Press any character to stop)

#Quick access to the ascii table.
man ascii

#Shutdown a Windows machine from Linux
net rpc shutdown -I ipAddressOfWindowsPC -U username%password
#This will issue a shutdown command to the Windows machine. username must be an administrator on the Windows machine. Requires samba-common package installed. Other relevant commands are:
net rpc shutdown -r : reboot the Windows machine
net rpc abortshutdown : abort shutdown of the Windows machine
#Type to show all relevant commands:
net rpc

#Jump to a directory, execute a command and jump back to current dir
(cd /tmp && ls)

#Display the top ten running processes – sorted by memory usage
ps aux | sort -nk +4 | tail

ps #returns all running processes which are then sorted by the 4th field in numerical order and the top 10 are sent to STDOUT.

#List of commands you use most often
history | awk ‘{a[$2]++}END{for(i in a){print a[i] ” ” i}}’ | sort -rn | head

#Reboot machine when everything is hanging (raising a skinny elephant)
<alt> + <print screen/sys rq> + <R> – <S> – <E> – <I> – <U> – <B>
#If the machine is hanging and the only help would be the power button, this key-combination will help to reboot your machine (more or less) gracefully.
## R – gives back control of the keyboard
## S – issues a sync
## E – sends all processes but init the term singal
## I – sends all processes but init the kill signal
## U – mounts all filesystem ro to prevent a fsck at reboot
## B – reboots the system
#Save your file before trying this out, this will reboot your machine without warning!
#http://en.wikipedia.org/wiki/Magic_SysRq_key

#Make ‘less’ behave like ‘tail -f’
less +F somelogfile
##Using +F will put less in follow mode. This works similar to ‘tail -f’. To stop scrolling, use the interrupt. Then you’ll get the normal benefits of less (scroll, etc.).
##Pressing SHIFT-F will resume the ‘tailling’.

#Set audible alarm when an IP address comes online
ping -i 60 -a IP_address
#Waiting for your server to finish rebooting? Issue the command above and you will hear a beep when it comes online. The -i 60 flag tells ping to wait for 60 seconds between ping, putting less strain on your system. Vary it to your need. The -a flag tells ping to include an audible bell in the output when a package is received (that is, when your server comes online).

#Backticks are evil
echo “The date is: $(date +%D)”

#This is a simple example of using proper command nesting using $() over “. There are a number of advantages of $() over backticks. First, they can be easily nested without escapes:
program1 $(program2 $(program3 $(program4)))versus
program1 `program2 \`program3 \`program4\`\``Second, they’re easier to read, then trying to decipher the difference between the backtick and the singlequote: `’. The only drawback $() suffers from is lack of total portability. If your script must be portable to the archaic Bourne shell, or old versions of the C-shell or Korn shell, then backticks are appropriate, otherwise, we should all get into the habit of $(). Your future script maintainers will thank you for producing cleaner code.

#Simulate typing
echo “You can simulate on-screen typing just like in the movies” | pv -qL 10
This will output the characters at 10 per second.

##python smtp server
python -m smtpd -n -c DebuggingServer localhost:1025
This command will start a simple SMTP server listening on port 1025 of localhost. This server simply prints to standard output all email headers and the email body.

#Watch Network Service Activity in Real-time
lsof -i

#diff two unsorted files without creating temporary files
diff <(sort file1) <(sort file2)
#bash/ksh subshell redirection (as file descriptors) used as input to diff

#Rip audio from a video file.
mplayer -ao pcm -vo null -vc dummy -dumpaudio -dumpfile <output-file> <input-file>
##replace accordingly

#Matrix Style
tr -c “[:digit:]” ” ” < /dev/urandom | dd cbs=$COLUMNS conv=unblock | GREP_COLOR=”1;32″ grep –color “[^ ]“
#This command will show you all the string (plain text) values in ram
sudo dd if=/dev/mem | cat | strings
#A fun thing to do with ram is actually open it up and take a peek.

#Display which distro is installed
cat /etc/issue

#Easily search running processes (alias).
alias ‘ps?’='ps ax | grep ‘

#Create a script of the last executed command
echo “!!” > foo.sh
#Sometimes commands are long, but useful, so it’s helpful to be able to make them permanent without having to retype them. An alternative could use the history command, and a cut/sed line that works on your platform.
history -1 | cut -c 7- > foo.sh

#Extract tarball from internet without local saving
wget -qO – “http://www.tarball.com/tarball.gz” | tar zxvf -

#Create a backdoor on a machine to allow remote connection to bash
nc -vv -l -p 1234 -e /bin/bash
#This will launch a listener on the machine that will wait for a connection on port 1234. When you connect from a remote machine with something like :
nc 192.168.0.1 1234
#Force remove package
sudo dpkg --force-all -r lighttpd
#You will have console access to the machine through bash. (becareful with this one)
************************************************************************

#Monitor progress of a command
pv access.log | gzip > access.log.gz
#Pipe viewer is a terminal-based tool for monitoring the progress of data through a pipeline. It can be inserted into any normal pipeline between two processes to give a visual indication of how quickly data is passing through, how long it has taken, how near to completion it is, and an estimate of how long it will be until completion. Source: http://www.catonmat.net/blog/unix-utilities-pipe-viewer/

#Graphical tree of sub-directories
ls -R | grep ":$" | sed -e 's/:$//' -e 's/[^-][^\/]*\//--/g' -e 's/^/   /' -e 's/-/|/'
#Prints a graphical directory tree from your current directory

#Delete all files in a folder that don’t match a certain file extension
rm !(*.foo|*.bar|*.baz)
#Deletes all files in a folder that are NOT *.foo, *.bar or *.baz files. Edit the pattern inside the brackets as you like.

#Easy and fast access to often executed commands that are very long and complex.
some_very_long_and_complex_command # label
#When using reverse-i-search you have to type some part of the command that you want to retrieve. However, if the command is very complex it might be difficult to recall the parts that will uniquely identify this command. Using the above trick it’s possible to label your commands and access them easily by pressing ^R and typing the label (should be short and descriptive).

#Define a quick calculator function
? () { echo "$*" | bc -l; }
#defines a handy function for quick calculations from cli.
#once defined:
? 10*2+3

#Display a cool clock on your terminal
watch -t -n1 "date +%T|figlet"
#This command displays a clock on your terminal which updates the time every second. Press Ctrl-C to exit.
#A couple of variants:
#A little bit bigger text:
watch -t -n1 "date +%T|figlet -f big"You can try other figlet fonts, too.
#Big sideways characters:
watch -n 1 -t '/usr/games/banner -w 30 $(date +%M:%S)'This requires a particular version of banner and a 40-line terminal or you can adjust the width (“30″ here).

#intercept stdout/stderr of another process
strace -ff -e trace=write -e write=1,2 -p SOME_PID

#Remove duplicate entries in a file without sorting.
awk '!x[$0]++' <file>
#Using awk, find duplicates in a file without sorting, which reorders the contents. awk will not reorder them, and still find and remove duplicates which you can then redirect into another file.

#Record a screencast and convert it to an mpeg
ffmpeg -f x11grab -r 25 -s 800x600 -i :0.0 /tmp/outputFile.mpg
#Grab X11 input and create an MPEG at 25 fps with the resolution 800×600

#Mount a .iso file in UNIX/Linux
mount /path/to/file.iso /mnt/cdrom -oloop
#“-o loop” lets you use a file as a block device

#Insert the last command without the last argument (bash)
!:-
/usr/sbin/ab2 -f TLS1 -S -n 1000 -c 100 -t 2 http://www.google.com/then

#!:- http://www.urfix.com/is the same as
/usr/sbin/ab2 -f TLS1 -S -n 1000 -c 100 -t 2 http://www.urfix.com/

#Convert seconds to human-readable format
date -d@1234567890
#This example, for example, produces the output, “Fri Feb 13 15:26:30 EST 2009″

#Job Control
^Z $bg $disown
#You’re running a script, command, whatever.. You don’t expect it to take long, now 5pm has rolled around and you’re ready to go home… Wait, it’s still running… You forgot to nohup it before running it… Suspend it, send it to the background, then disown it… The ouput wont go anywhere, but at least the command will still run…

#Edit a file on a remote host using vim
vim scp://username@host//path/to/somefile

#Monitor the queries being run by MySQL
watch -n 1 mysqladmin --user=<user> --password=<password> processlist
#Watch is a very useful command for periodically running another command – in this using mysqladmin to display the processlist. This is useful for monitoring which queries are causing your server to clog up.
#More info here: http://codeinthehole.com/archives/2-Monitoring-MySQL-processes.html

#escape any command aliases
\[command]
#e.g. if rm is aliased for ‘rm -i’, you can escape the alias by prepending a backslash:
rm [file] # WILL prompt for confirmation per the alias
\rm [file] # will NOT prompt for confirmation per the default behavior of the command

#Show apps that use internet connection at the moment. (Multi-Language)
ss -p
#for one line per process:
ss -p | cat #for established sockets only:
ss -p | grep STA #for just process names:
ss -p | cut -f2 -sd\"
##or##
ss -p | grep STA | cut -f2 -d\"

#Send pop-up notifications on Gnome
notify-send ["<title>"] "<body>"
#The title is optional.
#Options:
## -t: expire time in milliseconds.
## -u: urgency (low, normal, critical).
## -i: icon path.

#On Debian-based systems you may need to install the ‘libnotify-bin’ package.
#Useful to advise when a wget download or a simulation ends. Example:
wget URL ; notify-send "Done"

#quickly rename a file
mv filename.{old,new}

#Remove all but one specific file
rm -f !(survivior.txt)

#Generate a random password 30 characters long
strings /dev/urandom | grep -o '[[:alnum:]]' | head -n 30 | tr -d '\n'; echo
#Find random strings within /dev/urandom. Using grep filter to just Alphanumeric characters, and then print the first 30 and remove all the line feeds.

# Run a command only when load average is below a certain threshold
echo "rm -rf /unwanted-but-large/folder" | batch
#Good for one off jobs that you want to run at a quiet time. The default threshold is a load average of 0.8 but this can be set using atrun.

#Binary Clock
watch -n 1 'echo "obase=2;`date +%s`" | bc'
#Create a binary clock.

#Processor / memory bandwidthd? in GB/s
dd if=/dev/zero of=/dev/null bs=1M count=32768
#Read 32GB zero’s and throw them away.
#How fast is your system?

#Backup all MySQL Databases to individual files
for I in $(mysql -e 'show databases' -s --skip-column-names); do mysqldump $I | gzip > "$I.sql.gz"; done

##reddit bookmarklet
javascript:%20var%20x=%20$(".content").find("a").each(function(){var%20href=$(this).attr("href");if((!$(this).hasClass("drowsapMorphed"))%20&&%20($(this).next(".drowsapMorphed").length==0)%20&&%20href%20&&%20(href.indexOf('imgur')>=0%20||%20href.indexOf('jpeg')>=0%20||%20href.indexOf('jpg')>=0%20%20||%20href.indexOf('png')>=0)){var%20ext%20=(href.indexOf('imgur')>=0%20&&%20href.indexOf('jpg')<0%20&&%20href.indexOf('png')<0)%20?%20'.jpg'%20:'';%20var%20img%20=%20$("<a%20class='drowsapMorphed'%20href='"+href+"'%20target='blank'%20style='display:block'><img%20style='display:block;max-width:780px;'%20src='"+href+%20ext+"'%20/></a>");$(this).after(img);}});