Poor Mans SOS Report

sos.sh

#!/bin/bash
#set -x
#set -o verbose

export LANG=C

SOS_OUTPUT_DIR="/tmp/sos"
"${ERROR_LOGFILE}"FILE="error.log"

[[ -d "${SOS_OUTPUT_DIR}" ]] && rm -rf "${SOS_OUTPUT_DIR}"
mkdir "${SOS_OUTPUT_DIR}" && cd "${SOS_OUTPUT_DIR}" && mkdir -p  var/log etc/lvm etc/sysconfig network storage sos_commands/networking

echo -e "Gathering system information..."
hostname &> hostname.txt
cp -a /etc/redhat-release  ./etc/ 2>> "${"${ERROR_LOGFILE}"FILE}"
uptime &> uptime.txt

echo -e "Gathering application information..."
chkconfig --list &> chkconfig
top -bn1 &> top_bn1.txt
service --status-all &> service_status_all.txt
date &> date.txt
ps auxww &> ps_auxww.txt
ps -elf &> ps_-elf.txt
rpm -qa --last &> rpm-qa.txt
echo -e "Running 'rpm -Va'. This may take a moment."
rpm -Va &> rpm-Va.txt

echo -e "Gathering memory information..."
free -m &> free.txt
vmstat 1 10 &> vmstat.txt

echo -e "Gathering network information..."
ifconfig &> ./network/ifconfig.txt
netstat -s &>./network/netstat_-s.txt
netstat -agn &> ./network/netstat_-agn.txt
netstat -neopa &> ./network/netstat_-neopa.txt
route -n &> ./network/route_-n.txt

for i in $(ls /etc/sysconfig/network-scripts/{ifcfg,route,rule}-*) ; do echo -e "$i\n----------------------------------"; cat $i;echo " ";  done &> ./sos_commands/networking/ifcfg-files    

for i in $(ifconfig | grep "^[a-z]" | cut -f 1 -d " "); do 
  echo -e "$i\n-------------------------" ; 
  ethtool $i; 
  ethtool -k $i; 
  ethtool -S $i; 
  ethtool -i $i; 
  echo -e "\n" ; 
done &> ./sos_commands/networking/ethtool.out

cp /etc/sysconfig/network ./sos_commands/networking/ 2>> "${ERROR_LOGFILE}"

cp /etc/sysconfig/network-scripts/ifcfg-* ./sos_commands/networking/ 2>> "${ERROR_LOGFILE}"

cp /etc/sysconfig/network-scripts/route-* ./sos_commands/networking/ 2>> "${ERROR_LOGFILE}"

cat /proc/net/bonding/bond* &> ./sos_commands/networking/proc-net-bonding-bond 2>> "${ERROR_LOGFILE}"

iptables --list --line-numbers &> ./sos_commands/networking/iptables_--list_--line-numbers.txt

ip route show table all &> ./sos_commands/networking/ip_route_show_table_all.txt
ip link &> ./sos_commands/networking/ip_link.txt

echo -e "Gathering Storage/Filesystem information..."
df -l &> df
fdisk -l &> fdisk
parted -l &> parted
cp -a /etc/fstab  ./etc/ 2>> "${ERROR_LOGFILE}"
cp -a /etc/lvm/lvm.conf ./etc/lvm/ 2>> "${ERROR_LOGFILE}"
cp -a /etc/lvm/backup/ ./etc/lvm/ 2>> "${ERROR_LOGFILE}"
cp -a /etc/lvm/archive/ ./etc/lvm/ 2>> "${ERROR_LOGFILE}"
cp -a /etc/multipath.conf ./etc/ 2>> "${ERROR_LOGFILE}"
cat /proc/mounts &> mount  
iostat -tkx 1 10 &> iostat_-tkx_1_10
parted -l &> storage/parted_-l
vgdisplay -v &> storage/vgdisplay
lvdisplay &> storage/lvdisplay
pvdisplay &> storage/pvdisplay
pvs -a -v &> storage/pvs
vgs -v &> storage/vgs
lvs -o +devices &> storage/lvs
multipath -v4 -ll &> storage/multipath_ll
pvscan -vvvv &> storage/pvscan
vgscan -vvvv &> storage/vgscan
lvscan -vvvv &> storage/lvscan
lsblk &> storage/lsblk
lsblk -t &> storage/lsblk_t
dmsetup info -C &> storage/dmsetup_info_c
dmsetup status &>  storage/dmsetup_status 
dmsetup table &>  storage/dmsetup_table
ls -lahR /dev &> storage/dev

echo -e "Gathering kernel information..."
cp -a /etc/security/limits.conf ./etc/ 2>> "${ERROR_LOGFILE}"
cp -a /etc/sysctl.conf ./etc/ 2>> "${ERROR_LOGFILE}"
ulimit -a &> ulimit
cat /proc/slabinfo &> slabinfo
cat /proc/interrupts &> interrupts 
cat /proc/iomem &> iomem
cat /proc/ioports &> ioports
slabtop -o &> slabtop_-o
uname -a &> uname
sysctl -a &> sysctl_-a
lsmod &> lsmod
cp -a /etc/modprobe.conf ./etc/ 2>> "${ERROR_LOGFILE}"
cp -a  /etc/sysconfig/* ./etc/sysconfig/ 2>> "${ERROR_LOGFILE}"

for module in $(lsmod | grep -v "Used by"| awk '{ print $1 }'); do 
  modinfo  $module &>> modinfo;
done;

ipcs -a &> ipcs_-a.txt
ipcs -s | awk '/^0x/ {print $2}' | while read semid; do 
  ipcs -s -i $semid; 
done &> ipcs_-s_verbose.txt

sar -A &> sar_-A.txt
cp -a /var/log/dmesg dmesg 2>> "${ERROR_LOGFILE}"
dmesg &> dmesg_now.txt

echo -e "Gather hardware information..."
dmidecode &> dmidecode.txt
lspci -vvv &> lspci_-vvv.txt
lspci &> lspci.txt
cat /proc/meminfo &> proc_meminfo.txt
cat /proc/cpuinfo &> proc_cpuinfo.txt

echo -e "Gather kump information..."
cp -a /etc/kdump.conf ./etc/ 2>> "${ERROR_LOGFILE}"
ls -laR /var/crash &> ls-lar-var-crash
ls -1 /var/crash | while read n; do mkdir -p var/crash/${n}; cp -a /var/crash/${n}/vmcore-dmesg* var/crash/${n}/ 2>> "${ERROR_LOGFILE}"; done

echo -e "Gathering logs..."
cp -a /var/log/* ./var/log/ 2>> "${ERROR_LOGFILE}"
cp -a /etc/*syslog.conf ./etc/ 2>> "${ERROR_LOGFILE}"

echo -e "Compressing files..."
tar -cjf "${SOS_OUTPUT_DIR}"report.tar.bz2 ./

echo -e "Script complete."

sosreport.sh

#!/bin/bash

export LANG=C
export PATH=/sbin:/usr/sbin:/bin:/usr/bin

# If this script hangs, un-comment the below two entries and note the command that the script hangs on.  Then comment out that command and re-run the script.
# set -x
# set -o verbose

[[ -d /tmp/sosreport ]] && rm -rf /tmp/sosreport
mkdir -p /tmp/sosreport || (echo "WARNING: failed to create sosreport dir." && exit 1)
cd /tmp/sosreport && mkdir -p var/log etc/lvm etc/security etc/systemd etc/udev usr/lib/udev usr/lib/systemd network storage kernel proc draft

LOGFILE=/tmp/sosreport/error.log

echo -e "Gathering system information..."
cat /etc/redhat-release > etc/redhat-release 2>>$LOGFILE
cat /etc/localtime > etc/localtime 2>>$LOGFILE
date &> date
uptime &> uptime
pwd &> pwd
env &> env
command -- hostnamectl status &> hostnamectl
command -- timedatectl status &> timedatectl

echo -e "Gathering application and process information..."
ps auxww &> ps_auxww
ps axlfwww &> ps_axlfwww
ps -elfL &> ps_-elfL
ps axo pid,ppid,user,group,lwp,nlwp,start_time,comm,cgroup &> ps_group
ps axo flags,state,uid,pid,ppid,pgid,sid,cls,pri,addr,sz,wchan:20,lstart,tty,time,cmd &> ps_sched
top -bn1 &> top_-bn1
command -- lsof -b +M -n -l -P &> lsof_-bnlP.M
command -- chkconfig --list &> chkconfig
command -- service --status-all &> service_--status-all
ls -la /etc/alternatives/* &> ls_etc_alternatives_-la

echo -e "Gathering package information..."
cp -a /etc/rhsm etc/ 2>>$LOGFILE
cp -a /etc/yum* etc/ 2>>$LOGFILE
command -- rpm -qa --last &> installed-rpms
command -- rpm --nodigest -qa --qf '%{NAME}~~%{ARCH}~~%{VERSION}~~%{RELEASE}~~%{INSTALLTIME:date}~~%{INSTALLTIME}~~%{VENDOR}~~%{BUILDHOST}~~%{SIGPGP}~~%{SIGPGP:pgpsig}\n' &> rpm_info
echo -e "  * Going to run 'rpm -Va'. This can take a while."
command -- rpm -Va &> rpm_-Va

echo -e "Gathering basic service information..."
cp -a /etc/audit/  etc/ 2>>$LOGFILE
cp -a /etc/audisp/  etc/ 2>>$LOGFILE
cp -a /etc/*syslog.* etc/ 2>>$LOGFILE
cp -a /etc/logrotate.* etc/ 2>>$LOGFILE

echo -e "Gathering idm information..."
cp -a /etc/pam.d/ etc/ 2>>$LOGFILE
cp -a /etc/authselect/ etc/ 2>>$LOGFILE
cp -a /etc/nsswitch.conf etc/ 2>>$LOGFILE
ls -lanR /etc/krb5* /etc/yp.conf /etc/sssd /etc/samba &> ls_idm

echo -e "Gathering memory information..."
free -m &> free  
command -- vmstat 1 10 &> vmstat

echo -e "Gathering network information..."
cp -a /etc/NetworkManager/ etc/NetworkManager/ 2>>$LOGFILE
cat /proc/net/tcp /proc/net/tcp6 &> proc/net_tcp
cat /proc/net/udp /proc/net/udp6 &> proc/net_udp
cat /proc/net/unix &> proc/net_unix
cat /proc/net/bonding/bond* &> proc/net_bonding_bond_all 2>>$LOGFILE
command -- ifconfig -a &> network/ifconfig_-a
command -- netstat -s &> network/netstat_-s
command -- netstat -agn &> network/netstat_-agn
command -- netstat -neopa &> network/netstat_-neopa
command -- route -n &> network/route_-n
command -- ip -o addr &> network/ip_-o_addr
command -- ip address &> network/ip_-d_address
command -- ip -s -d link &> network/ip_-s_-d_link
command -- ip -4 route show table all &> network/ip_-4_route_show_table_all
command -- ip -6 route show table all &> network/ip_-6_route_show_table_all
command -- ip -4 rule &> network/ip_-4_rule
command -- ip -6 rule &> network/ip_-6_rule
command -- ip mroute show &> network/ip_mroute_show
command -- ip maddr show &> network/ip_maddr_show
command -- ip -s -s neigh show &> network/ip_-s_-s_neigh_show
command -- ip neigh show nud noarp &> network/ip_neigh_show_nud_noarp
command -- tc -s qdisc show &> network/tc_-s_qdisc_show
command -- ss -neopaim &> network/ss_-neopaim
command -- iptables --list --line-numbers &> network/iptables_--list_--line-numbers
command -- iptables -vnxL &> network/iptables_-vnxL
command -- ip6tables -vnxL &> network/ip6tables_-vnxL
command -- iptables-save &> network/iptables-save
command -- ifenslave -a &> network/ifenslave_-a
command -- biosdevname -d &> network/biosdevname_-d
for i in /sys/class/net/*; do ndev=`basename $i`; echo "###### $ndev";
  ethtool $ndev;
  echo "###-d"
  ethtool -d $ndev;
  echo "###-i"
  ethtool -i $ndev;
  echo "###-k"
  ethtool -k  $ndev;
  echo "###-S"
  ethtool -S  $ndev;
  echo "###-T"
  ethtool -T  $ndev;
  echo "###-a"
  ethtool -a  $ndev;
  echo "###-c"
  ethtool -c  $ndev;
  echo "###-g"
  ethtool -g  $ndev;
  echo "###-P"
  ethtool -P  $ndev;
  echo "###-l"
  ethtool -l  $ndev;
  echo "###--phy-statistics"
  ethtool --phy-statistics  $ndev;
  echo "###--show-priv-flags"
  ethtool --show-priv-flags  $ndev;
  echo "###--show-eee"
  ethtool --show-eee  $ndev;
done &> network/ethtool_all

echo -e "Gathering storage/filesystem information..."
cp -a /etc/fstab  etc/ 2>>$LOGFILE
cp -a /etc/lvm/lvm.conf etc/lvm/ 2>>$LOGFILE
cp -a /etc/lvm/backup/ etc/lvm/ 2>>$LOGFILE
cp -a /etc/lvm/archive/ etc/lvm/ 2>>$LOGFILE
cp -a /etc/multipath.conf etc/ 2>>$LOGFILE
cat /proc/mounts &> proc/mounts
cat /proc/self/mountinfo &> proc/self_mountinfo
cat /proc/1/mountinfo &> proc/1_mountinfo
cat /proc/scsi/scsi &> proc/scsi_scsi
df -l &> storage/df_-l
fdisk -l &> storage/fdisk_-l
command -- parted -sl &> storage/parted_-sl
command -- iostat -tkx 1 10 &> storage/iostat_-tkx_1_10
vgdisplay -v &> storage/vgdisplay_-v
lvdisplay &> storage/lvdisplay
pvdisplay &> storage/pvdisplay
pvs -av &> storage/pvs_-av
vgs -v &> storage/vgs_-v
lvs -o +devices &> storage/lvs_-o.devices
pvscan -vvvv &> storage/pvscan
vgscan -vvvv &> storage/vgscan
lvscan -vvvv &> storage/lvscan
lsblk &> storage/lsblk
lsblk -t &> storage/lsblk_-t
blkid -c /dev/null &> storage/blkid_-c
command -- multipath -v4 -ll &> storage/multipath_-v4_-ll
dmsetup info -C &> storage/dmsetup_info_-C
dmsetup status &> storage/dmsetup_status
dmsetup table &> storage/dmsetup_table
ls -lanR /dev &> storage/ls_dev_-lanR
udevadm info --export-db &> storage/udevadm_info_--export-db
command -- lsscsi -dg &> storage/lsscsi_dg

echo -e "Gathering kernel information..."
cp -a /etc/security/limits* etc/security/ 2>>$LOGFILE
cp -a /etc/sysctl* etc/ 2>>$LOGFILE
cp -a /etc/modprobe* etc/ 2>>$LOGFILE
cp -a /etc/module* etc/ 2>>$LOGFILE
cp -a /etc/sysconfig/ etc/ 2>>$LOGFILE
cp -a /var/log/dmesg var/log/dmesg 2>>$LOGFILE
cat /proc/slabinfo &> proc/slabinfo
cat /proc/interrupts &> proc/interrupts 
cat /proc/iomem &> proc/iomem
cat /proc/ioports &> proc/ioports
cat /proc/cmdline &> proc/cmdline
command -- tar cf ./kernel/$(uname -r)_mod.tar /lib/modules/$(uname -r)/modules* 2>>$LOGFILE
ulimit -a &> ulimit_-a
slabtop -o &> slabtop_-o
uname -a &> uname
sysctl -a &> sysctl_-a
lsmod &> lsmod
for MOD in `lsmod | grep -v "Used by"| awk '{ print $1 }'`; do modinfo $MOD 2>&1 >> modinfo_all; done;
ipcs -a &> ipcs_-a
ipcs -s | awk '/^0x/ {print $2}' | while read semid; do ipcs -s -i $semid; done &> ipcs_-s_verbose
command -- sar -A &> sar_-A
dmesg &> dmesg_now
dmesg -T &> dmesg_-T

echo -e "Gathering hardware information..."
cat /proc/meminfo &> proc/meminfo
cat /proc/cpuinfo &> proc/cpuinfo
command -- dmidecode &> dmidecode
command -- lspci -vvv &> lspci_-vvv
command -- lspci -n &> lspci_-n
command -- lshw -json &> lshw_-json

echo -e "Gathering kdump information..."
cp -a /etc/kdump.conf etc/ 2>>$LOGFILE
ls -lanR /var/crash &> ls_var_crash_-lanR
ls -1 /var/crash | while read n; do mkdir -p var/crash/${n}; cp -a /var/crash/${n}/vmcore-dmesg* var/crash/${n}/ 2>>$LOGFILE; done

echo -e "Gathering boot information..."
cp -a /etc/grub* etc/ 2>>$LOGFILE
ls -lanR /boot &> ls_boot_-lanR
command -- lsinitrd &> lsinitrd
command -- grubby --info=ALL &> grubby_info

echo -e "Gathering aide information..."
cp -a /etc/aide.conf etc/ 2>>$LOGFILE
cp -a /var/log/aide var/log/ 2>>$LOGFILE
ls -lanR /var/lib/aide &> ls_var_lib_aide_-lanR

echo -e "Gathering abrt information..."
cp -a /etc/abrt etc/ 2>>$LOGFILE
ls -lanR /var/spool/abrt &> ls_var_spool_abrt_-lanR

echo -e "Gathering container related information..."
mkdir container
if command -v podman &>/dev/null; then
    podman ps | awk '$1!="CONTAINER" {print $1}' | while read id; do podman inspect $id &> container/inspect_${id}; done
    podman ps &> container/ps
    podman image list &> container/image_list
fi
type docker &> container/type_docker

echo -e "Gathering library information..."
cp -a /etc/ld.so.conf* etc/ 2>>$LOGFILE
cp -a /etc/ld.so.preload etc/ 2>>$LOGFILE
ldconfig -p -N -X &> ldconfig_-p_-N_-X

echo -e "Gathering python information..."
type python &> type_python
ls -la /usr/bin/python* > python_bininfo

echo -e "Gathering systemd information..."
cp -a /etc/systemd etc/ 2>>$LOGFILE
cp -a /usr/lib/systemd/*.conf usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/network usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/portable usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/system usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/system-preset usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/system usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/user/ usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/user-preset/ usr/lib/systemd/ 2>>$LOGFILE
cp -a /usr/lib/systemd/system-shutdown/ usr/lib/systemd/ 2>>$LOGFILE
cp -a /etc/udev/udev.conf etc/udev/ 2>>$LOGFILE
cp -a /etc/udev/rules.d/ etc/udev/ 2>>$LOGFILE
cp -a /usr/lib/udev/rules.d/ usr/lib/udev/ 2>>$LOGFILE
command -- systemctl status --all --full --no-pager &> systemctl_status

echo -e "Gathering logs..."
cp -a /var/log/{containers*,message*,secure*,boot*,cron*,yum*,Xorg*,sa,rhsm,audit} var/log/ 2>>$LOGFILE
command -- journalctl --no-pager --all --full --boot > var/log/journalctl_now 2>>$LOGFILE

echo -e "Adjusting symlinks in report..."
find etc/ -type l -exec bash -c "for f in '{}'; do TMPFILE=\`mktemp -p . tmp_XXXXXX\` && [[ $? -eq 0 ]] && echo -n '## FILE: ' > \"\${TMPFILE}\" && ls -la \"\${f}\" >> \"\${TMPFILE}\" 2>&1 && cat \"\${f}\" >> \"\${TMPFILE}\" 2>&1 && mv -f \"\${TMPFILE}\" \"\${f}\"; done" \;
mv tmp_* draft/

echo -e "Compressing files..."
OUTFILE=/tmp/mansos-$(uname -n)-$(date +%s).tar.gz
if command -v tar &>/dev/null; then
    echo -e "  * Saving to: ${OUTFILE}"
    tar -zpcf "${OUTFILE}" ./
else
    echo "WARNING: tar is not available. Please submit the /tmp/sosreport directory instead."
fi

echo -e "Script complete."