Bash Cheatsheet

linux-cheatsheet.sh

# Check which process is using a port.
lsof -i tcp:57454

# Bash scrict mode.
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'

# Exit script if you try to use an uninitialized variable.
set -o nounset

# Exit script if a statement returns a non-true return value.
set -o errexit

# Use the error status of the first failure rather than last item in a pipe.
set -o pipefail

# Subshell example.
(cd /tmp && ls -la)

# Show the difference between two files.
comm -23 file1.txt file2.txt

# Expansion (no spaces).
mv myfile.{js,jsx}
touch myfile.{html,css,js,test.js,stories.js}

# Stripping out prefix and suffix.
var=hello.pdf
echo ${var%.pdf} # hello
echo ${var#hello} # .pdf

# Default values to variables
var=${1:-hello.pdf}

# Argument expansion (always quote variables).
set -- "arg  1" "arg  2" "arg  3"

for word in "$*"; do echo "$word"; done
arg  1 arg  2 arg  3

for word in "$@"; do echo "$word"; done
arg  1
arg  2
arg  3

# If statement with regex matching.
if [[ "$ENV" =~ ^('production'|'staging')$ ]]; then
  echo "$ENV"
fi

# See exp branches with master but not staging.
# Requires git 2.13, circle-ci uses git 2.11
exp_branches_outdated=$(git branch -r --no-contains remotes/origin/staging --contains remotes/origin/master -- *exp/*)

# Find all folders with git repos.
find . -maxdepth 3 -name .git -type d | rev | cut -c 6- | rev

# c is a union b
sort a b | uniq > c

# c is a intersect b
sort a b | uniq -d > c

# c is set difference a - b
sort a b b | uniq -u > c

# Create symbolic link / update.
ln -s /path/to/file /path/to/symlink
ln -sf /path/to/file /path/to/symlink

# Example symbolic link to usr/local/bin.
sudo ln -s /usr/local/android-studio/bin/studio.sh /usr/local/bin/android-studio

# Find location of installed packages and headers.
dpkg -L opencl-headers

# Play sounds remotely
ssh user@host 'mpg321 -' < sounds.mp3

# xdotool keystrokes
xdotool key --repeat 100 --repeat-delay 200 ctrl+alt+Right
xdotool key --repeat 100 --repeat-delay 200 ctrl+alt+Right --window WINDOW_ID

# xdotool getmouselocation to get windowid and mouse position.
xdotool getmouselocation
xdotool mousemove 100 100

# TODO: Move in circle
# Needs translation: (x - h)^2 + (y - k)^2 = r^2  
for x in $(seq 540 1 810); do bc -l <<< "sqrt(270^2 - $x^2)" | xargs xdotool mousemove "$x $1"; done

# Generate self-signed certificate.
openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes

# Delete all merged branches to master.
# http://stevenharman.net/git-clean-delete-already-merged-branches
git branch --merged | grep -v "\*" | xargs -n1 git branch -d

# Git pull all repos (SSH keys).
ls -d -- */ | xargs -I {} git -C {} pull
ls -d -- */ | xargs -I {} git -C {} push

# Git pull all.
find . -maxdepth 3 -name .git -type d | rev | cut -c 6- | rev | xargs -I {} git -C {} pull

# Git blame while ignoring whitespace and moved files.
git blame -w -M

# Commits since last tag.
git log $(git tag | tail -1)..HEAD --no-merges --pretty='[%h] %s'
git log $(git tag | tail -1)..HEAD --no-merges --pretty='%s' | grep -E "\[.*\]"

# Clean everything.
git clean -fxd
yarn cache clean
yarn --force

# Git assume unchanged files & see which files are affected.
git update-index --assume-unchanged <file>
git update-index --no-assume-unchanged <file>
git ls-files -v | grep ^h

# Mkdir and cd in same line.
mkdir -p hello/world && cd $_

# While inside the directory tar files while excluding directories and files.
tar --exclude='.git*' --exclude='node_modules' -zcvf my-files.tgz .

# List by file size 
du -h interactives/ | gsort -h

# Find files bigger than 1GB.
find /path -size +1G

# Delete files that are older than 90 days.
find /path -type f -mtime +90 -exec rm -f {} +

# Change repo from using https to ssh.
ack 'https://[email protected]' --noignore-dir=.git
ack '[email protected]' --noignore-dir=.git
ack 'https://[email protected]' --noignore-dir=.git -l | xargs -n1 sed -i 's https://[email protected]/ [email protected]: g'

# Crop videos.
ffmpeg -i 2017_06_07_00_19_25.mp4 -acodec copy -vcodec copy -ss 0 -t 00:05:40 2017_06_07_00_19_25_clip.mp4

# Reuse flags and options in bash scripts using arrays.
exclude_flags=( --exclude node_modules --exclude .git --exclude bower_components )
rsync -a ../../Configuration/ Configuration "${exclude_flags[@]}"
rsync -a ../../Datatron/ Datatron "${exclude_flags[@]}"

# Create timestamps
for i in {1..3}; do echo $(($(date +%s) + $i)); done

# Create n files with timestamp prefixed.
now=$(date +%s); for i in {0..12}; do echo $(($now + $i))-CREATE-; done

# See exact package versions.
grep "version\"\:" client/components/*/.bower.json

# Automatically fix some errors with eslint.
git ls-files | grep -E .js$ | xargs eslint --fix

# Actually replace 4 spaces with 2 for js files.
git ls-files | grep -E .js$ | xargs -I{} sh -c 'unexpand -t 4 "$1" | expand -t 2 > "$1"' -- {}

# Add use strict to the top of every js file.
git ls-files | grep -E .js$ | xargs -n1 sed -i "1i 'use strict';\n"

# Replace 4 spaces with 2 in js. An example of excluding multiple folders with grep.
grep -E "^\s{4}" . -rl --include "*.js" --exclude={*components*,*node_modules*} | xargs sed -i "" -e "s/\s{4}/\s{2}/g"

# Puts space between closing paren and opening brace.
git ls-files | grep -E .js$ | xargs -n1 gsed -i -E 's/\)\{/\) \{/g;'

# Puts space between if, for, while and opening brace.
git ls-files | grep -E .js$ | xargs -n1 gsed -i -E 's/if\(/if \(/g; s/while\(/while \(/g; s/for\(/for \(/g;'

# Word count ignoring single occurrence: https://www.wordclouds.com/
egrep -o [a-zA-Z]+ $1 | tr [A-Z] [a-z] | sort | uniq -c | sort | awk '{$1=$1};1' | grep -Ev '^1 '

# Word count aggregation
egrep -o [a-zA-Z]+ docs/* | tr [A-Z] [a-z] | sort | uniq -c | sort | awk '{$1=$1};1' | grep -Ev '^1 '

linux-commands.sh

# Tagging commands as comments with reverse search (ctrl-r).
cd ~/Workspace/Datatron #datatron

# See list of API calls in repo.
grep -rEho "[\'\`]/api/v1.+[\'\`]" src/* --exclude *.test.js | sort -u

# Add tldr command for community simplified man pages.
https://github.com/tldr-pages/tldr

# Commit word counter.
words='shit fuck hack kludge crap damn oops dang darn stupid dumb ugh'
for word in $words; do echo $word `git log --oneline | grep -i $word | wc -l`; done

# Terminal Incognito
set +o history
set -o history

# Change history format.
HISTTIMEFORMAT='I ran this at: %d/%m/%y %T '

# QH: I do this too often.
git checkout dev && git pull && git checkout - && git rebase origin/dev

# Unix SQL-like join command.
# https://shapeshed.com/unix-join/
join authors.txt books.txt

# Specify field separator such as in csv.
join -t , authors.csv books.csv

# Abort other process.
kill -SIGABRT 666
kill -SIGINT 666

# Trigger out of memory due to async node.
node -e "while (true) process.stdout.write('');"


# Bash variable expansion: http://stackoverflow.com/a/8748880
${foo}bar
${array[42]}
${filename%.*}
$8 $9 ${10} ${11}

# Variable substring operation: http://stackoverflow.com/a/2059836
${string##substring}

# Fetch weather with curl
curl wttr.in

# Only on mac.
mdfind -name Sophos

# Unload Sophos daemon.
launchctl unload -w /Library/LaunchAgents/com.sophos.agent.plist
launchctl unload -w /Library/LaunchAgents/com.sophos.uiserver.plist

# Tail heroku logs
heroku logs -t -a stash-invest-edge-pr-<pull-request-number>

# Load testing with ApacheBench.
curl -X GET "http://localhost:8080"
ab -k -c 20 -n 250 "http://localhost:8080"

# Enter docker VM session
screen ~/Library/Containers/com.docker.docker/Data/com.docker.driver.amd64-linux/tty

# Rename command in debian based distros.
sudo dpkg-divert --local --divert /usr/bin/ack --rename --add /usr/bin/ack-grep

# AWS ec2 filter instances by name and group, output results as table.
aws ec2 describe-instances --filters "Name=tag:Group,Values=docker-swarm" "Name=tag:Swarm,Values=manager" --output table

# Take screenshots (MAC)
# http://osxdaily.com/2011/08/11/take-screen-shots-terminal-mac-os-x/
screencapture screenshot.jpg

# Replace all spaces with dashes in filenames. Use -n for dry run.
rename -n 's/\s+/-/g' *

# Move top 3 most recent files. Fails with files with whitespace so escape it.
ls -t | head -3 | sed 's/ /\\ /g' | xargs -n1 -I % mv % .intro/
mv $(ls -t | head -3) .intro/

# Returns status code of a local request
wget --spider -S "http://localhost" 2 > &1 | awk '/HTTP\// {print $2}'

# Using HTTPie
http localhost:8081 --check-status

# Run command with nohup and nice
nohup nice sudo -E /opt/python/run/venv/bin/python3 run.py setup --config Production &

# Read all timestamped SQL files.
ls | sort | xargs cat | pbcopy

# Show unique history of commands.
history | cut -c 8- | sort | uniq

# Show most used commands
history | cut -c 8- | sort | uniq -c | sort -n

# List all users in a unix system.
sudo sed 's|\([^:]*\).*|\1|g' /etc/passwd

# List out the columns in table.
cat table.txt | sed 1d | awk '{ print $2 }'

# Sum the columns in table.
cat table.txt | sed 1d | awk '{ sum += $2 } END { print sum }'

# Get list of io domains. Radio -> rad.io
cat | tr '[:upper:]' '[:lower:]' | sed -E 's/(.{2})$/\.\1/gi'

# Get word frequency of a file.
egrep -o [a-zA-Z]+ 98.txt | tr [a-z] [A-Z] | sort | uniq -c | sort

# Change hostname of server.
sudo hostnamectl set-hostname web-server-1

# hmac
echo -n "GET/api/project" | openssl dgst -sha256 -hmac "$(echo SHA256= | base64 --decode)"

# Show node log files
pm2 list
pm2 info <id>

# rsyslog and remote_syslog
sudo vim /etc/log_files.yml
sudo service rsyslog restart
sudo service remote_syslog restart

bash console

https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux

• Link contains color-mode (weight), text-mode (decorations), foreground, background.

Black        0;30     Dark Gray     1;30
Red          0;31     Light Red     1;31
Green        0;32     Light Green   1;32
Brown/Orange 0;33     Yellow        1;33
Blue         0;34     Light Blue    1;34
Purple       0;35     Light Purple  1;35
Cyan         0;36     Light Cyan    1;36
Light Gray   0;37     White         1;37

text_red='\033[0;31m'
text_green='\033[0;32m'
text_normal='\033[0m'

# echo -e enables blackslash interpretation
echo -e "${text_red}Please add env variable '$env_var' to app.json${text_normal}"

Getting rsyslog and remote_syslog to work with papertrail.

# SSH into server and edit this file.
sudo vim /etc/log_files.yml

files:
  - /home/ubuntu/.pm2/logs/*.log
destination:
  host: logs3.papertrailapp.com
  port: 19771
  protocol: tls
exclude_patterns:
  - GET \/\s
  - GET \/scripts
  - GET \/components
  - GET \/style

Linux Getty Virtual Console (Ctr+Alt+F3)
https://en.wikipedia.org/wiki/Getty_(Unix)

Python set log level of third party modules.

logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)

TensorFlow setup

# Long instructions:
# https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/

# Download neccessary images with googliser.
googliser --no-gallery --phrase "darth vader" --number 100 --lower-size 10000 --upper-size 5000000

# Create a folder to host the tf_files and the classification directories.
mkdir -p tf_files/characters

# Short-hand options. Manually delete undesired images.
googliser -g -p "darth vader" -n 50 -l 10000 -u 5000000
googliser -g -p "flower" -n 50 -l 10000 -u 5000000
googliser -g -p "hulk" -n 50 -l 10000 -u 5000000

# Install the TensorFlow docker container and exit with <ctr-d>.
sudo docker run -it gcr.io/tensorflow/tensorflow:latest-devel

# Mount the docker container with virtual volume.
sudo docker images 
sudo docker run -it -v $HOME/Workspace/tf_files:/tf_files c3efccc5f94f

# Inside the container, cd into the tensorflow directory and do a git pull.
cd /tensorflow/
git pull

# Retrain Inception with the new training set using 500 iterations. 
# Remove the `--how_many_training_steps` option to use a default of 4000 iterations. 
python tensorflow/examples/image_retraining/retrain.py \
--bottleneck_dir=/tf_files/bottlenecks \
--how_many_training_steps 500 \
--model_dir=/tf_files/inception \
--output_graph=/tf_files/retrained_graph.pb \
--output_labels=/tf_files/retrained_labels.txt \
--image_dir /tf_files/characters

Show help with EOF instead of multiline echo

show_help() {
cat << EOF
Usage: ${0##*/} [-h] [-t type]
Deploy code to remote machines

    -h --help         display this help and exit
    -t --type         the type of instance, eg. Postgres-Redshift
    -e --environment  the environment to deploy to ('Dev' or 'Prod') If not set, environment will be based on currently checked out branch ('develop' for Dev and 'master' for Prod)
EOF
}

Parsing command-line arguments without positions

while [[ $# > 0 ]]
do
    key="$1"

    case $key in
        -h|-\?|--help)
            show_help
            exit 0
            ;;
        -t|--type)
            TYPE="$2"
            shift # past argument
            ;;
        -e|--environment)
            ENV="$2"
            shift # past argument
            ;;
        *)
            # unknown option, skip
            ;;
    esac
    shift # past argument or value
done

Open slack app with developer options

user@MacOS $ pwd
/Applications/Slack.app/Contents/MacOS

user@MacOS $ SLACK_DEVELOPER_MENU=true ./Slack
2017-04-27 10:16:30.695 Slack[7660:117801] Creating Slack Application

Packaging Electon Application

https://electron.atom.io/docs/tutorial/application-packaging/

Upgrading NPM

npm install npm@latest -g

Samsung Kies

• https://www.samsung.com/us/support/owners/app/kies