useful unix commands

commands.txt

# display file with line numbers
cat file -n

# display gzipped file with line numbers
zcat file | cat -n

# display file from the i. line to j. line (that is j minus i there)
cat file | head -n j | tail -n j-i

# remove first line of the file (efficient)
tail -n +2 file > new_file

# remove first line of the file (less efficient)
sed 1d file > newfile

# count how many lines of a file contain a pattern
cat file | grep pattern | wc -l
grep pattern -c file

# select non-matching lines
cat file | grep -v pattern

# get the number of the first line where a pattern occur
cat file -n | grep pattern -m 1 | cut -f1 | tr -d ' '

# display N lines after, before, around pattern
grep -A N pattern file
grep -B N pattern file
grep -C N pattern file

# display N lines around pattern (only first appearance, useful if you know that pattern is unique)
grep -m 1 -C N pattern file

# adding together two (or more) files and create a new file from the result
cat file1 file2 > newfile

# create new compressed tar archive
tar -zcvf archive.tar.gz dirname

# extract from tar archive
tar -xzf archive.tar.gz

# view an tar archive
tar tvf archive.tar.gz

# extract tar.bz2
tar jxf filename.tar.bz2

# add user
sudo adduser <username>

# add group
sudo groupadd <groupname>

# add existing user to group
sudo usermod -a -G <groupname> <username>

# add a user to multiple groups
sudo usermod -a -G <groupname1>,<groupname2>,<groupname3> <username>

# view user's group assignemnts
id <username>
groups <username>

# make a directory accessible for a group
# change group of all files/directories recursively
sudo chgrp -R <groupname> <directory>
# add write permissions to group
sudo chmod -R g+w <directory>
# set "GID", so that all new files and directories created under <directory> are owned by the group
sudo find <directory> -type d -exec chmod 2775 {} \;

# randomize lines of a big file memory efficiently
uuid -v 4 -n $(wc -l < lines.txt) | paste - lines.txt | sort | cut -f2 > lines_random.txt 

# change hostname
sudo vim /etc/hostname -> change old name to new name
sudo vim /etc/hosts -> change old name to new name
sudo /etc/init.d/hostname restart

# search a device for bad blocks
sudo badblocks -v <device_name>
sudo badblocks -v /dev/md2

# renice (reset priority of processgroup): niceness: 0 (normal) - 19 (low)
sudo renice -n <niceness> -g <pid>

# count and filter csv files
# remove header (first line), the separator in csv file is ","; count the rows where the value in the second column is above a threshold value (4.5)
tail -n +2 file.csv | awk -F "," '$2 >= <my threshold value>' | wc -l

# pg_dump ssh tunnel
ssh -o "Compression=no" mydbserver "pg_dump -Fc -Z9 -U postgres mydb" > mydb.dump

# generate random password
# of course there are tons of ways to do this
# you can replace 32 with the desired password length
< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;

# force the user to change their password upon next login
sudo chage -d 0 <username>

# server shutdown
sudo shutdown -h now
sudo poweroff

# encrypt/decrypt file with openssl
openssl aes-256-cbc -salt -in <file> -out <encrypted_file> -k <passphrase>
openssl aes-256-cbc -d -in <encrypted_file> -out <file> -k <passphrase>

# generate your dhparam.pem file
openssl dhparam -out /etc/nginx/ssl/dhparam.pem 2048

# search for pattern in codebase
grep -inIEr --color=ALWAYS "<pattern_to_search>" .

# copy directory tree recursively without overwrite
rsync -a -v --ignore-existing <src_dir> <dst_dir>

# find processes with most memory usage
ps -e -orss,%mem,cputime,%cpu,pid,args | sort -b -k1,1n | pr -TW$COLUMNS

# find processes with maximum file descriptors
lsof -Fpcn | nawk '/^p/ { pid=substr($0,2) } /^c/ { cmd=substr($0,2) } /^n/ { fd[cmd"["pid"]"]++ } END  { for (cc in fd) printf("%-30s %i\n",cc,fd[cc]) } ' | sort -n -k 2 | tail -30

# copy file with rsync over SSH 
rsync -avz -e "ssh -i <ssh_key_path> -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress <file> <username>@<remote_ip>:<remote_dir>