fio stress test to test a whole block device in a loop and catching kernel & smart errors in-between to proof if the SATA bridge and Kernel is stable

fio_stability_loop.sh

#!/usr/bin/env bash
# Endless SATA/UASP stability soak with fio + SMART (robust shutdown)
# DESTROYS DATA on the target device. Use a WHOLE device (e.g. /dev/sdb, not /dev/sdb1).
# Usage: sudo ./fio_stability_loop.sh /dev/sdX
set -euo pipefail

#====================== configuration ======================
DEVICE="${1:-}"           # REQUIRED whole block device
SEQ_DURATION=300          # 5m seq read/write
RAND_DURATION=300         # 5m 4k 70/30
IODEPTH_SEQ=32
IODEPTH_RAND=32
RAND_NUMJOBS=4
RW_MIX_READ=70
BS_SEQ="1M"
BS_RAND="4k"
ENGINE="libaio"           # set to "io_uring" on mainline if desired
PIN_FIO_CORE=0
QUIET="${QUIET:-0}"       # QUIET=1 → less console spam, logs still captured
SMART_TIMEOUT="${SMART_TIMEOUT:-10}"  # seconds per smartctl call

ERROR_PAT='(ata[0-9]+: (hard resetting link|failed command|link down|COMRESET|SATA link down|frozen|status:)|exception Emask|I/O error|end_request: I/O error|Buffer I/O error|uas.*reset|usb .* reset|xHCI host controller not responding|NCQ.*(failed|error)|failed to IDENTIFY|device offlined)'

SMART_KEYS=("Power_On_Hours" "Power_Cycle_Count" "Reallocated_Sector_Ct" "Reported_Uncorrect"
  "Program_Fail_Count" "Erase_Fail_Count" "End-to-End_Error" "CRC_Error_Count"
  "UDMA_CRC_Error_Count" "Runtime_Bad_Block" "Total_Bad_Blocks"
  "Media_Wearout_Indicator" "Percentage_Used" "Temperature_Celsius" "Temperature_Internal")

#====================== singleton lock =====================
LOCK_FILE="/tmp/fio_stability_loop.lock"
exec 9>"$LOCK_FILE"
if ! flock -n 9; then
  echo "Another fio_stability_loop is already running (lock: $LOCK_FILE)."
  exit 1
fi

#====================== sanity checks ======================
if [[ -z "$DEVICE" || ! -b "$DEVICE" ]]; then
  echo "Usage: $0 /dev/sdX"; exit 1; fi
if [[ "$DEVICE" =~ [0-9]$ ]]; then
  echo "Refusing partition ($DEVICE); use the WHOLE device."; exit 1; fi

#====================== logging paths ======================
TS="$(date +%F_%H-%M-%S)"
BASE="stability_$(basename "$DEVICE")_${TS}"
LOG_DIR="./$BASE"; mkdir -p "$LOG_DIR"
FIO_LOG="$LOG_DIR/fio.log"
RUN_LOG="$LOG_DIR/run.log"
KERN_LOG="$LOG_DIR/kernel.log"
ERR_LOG="$LOG_DIR/errors.log"
SUMMARY="$LOG_DIR/summary.txt"
SMART_DIR="$LOG_DIR/smart"; mkdir -p "$SMART_DIR"
KLOG_PIDFILE="/tmp/fio_stability_klog.pid"

#====================== helpers (functions) =================
ts(){ date +"%F %T"; }
qecho(){ if [[ "$QUIET" == "1" ]]; then echo "$*" >>"$RUN_LOG"; else echo "$*" | tee -a "$RUN_LOG"; fi; }
qbegin(){ qecho "$(ts) :: BEGIN $*"; }
qend(){   qecho "$(ts) :: END   $*"; }

detect_smart_devflag(){
  if smartctl -i "$DEVICE" >/dev/null 2>&1; then echo ""
  elif smartctl -d sat -i "$DEVICE" >/dev/null 2>&1; then echo "-d sat"
  elif smartctl -d scsi -i "$DEVICE" >/dev/null 2>&1; then echo "-d scsi"
  else echo ""; fi
}
SMART_FLAG="$(detect_smart_devflag)"

# ARG-SAFE smart snapshot (works with set -u), bounded by timeout
smart_snapshot() {
  local _tag="${1-}"
  [[ -z "$_tag" ]] && _tag="snap_$(date +%F_%H-%M-%S)"
  local out="$SMART_DIR/smart_${_tag}.txt"
  timeout "$SMART_TIMEOUT" smartctl $SMART_FLAG -a "$DEVICE" >"$out" 2>&1 || true
  echo "$out"
}

smart_parse_raw(){
  awk -v OFS='=' '
    BEGIN{split("Power_On_Hours,Power_Cycle_Count,Reallocated_Sector_Ct,Reported_Uncorrect,Program_Fail_Count,Erase_Fail_Count,End-to-End_Error,CRC_Error_Count,UDMA_CRC_Error_Count,Runtime_Bad_Blocks,Total_Bad_Blocks,Media_Wearout_Indicator,Percentage_Used,Temperature_Celsius,Temperature_Internal",k,","); for(i in k) keys[k[i]]=1}
    { if($0~/Temperature_Celsius|Temperature_Internal|^Temperature:/){
        if($0~/Temperature_Celsius/) printf("Temperature_Celsius=%s\n",$NF);
        if($0~/Temperature_Internal/) printf("Temperature_Internal=%s\n",$NF);
        if($0~/^Temperature:/){t=$0; gsub(/[^0-9]/,"",t); if(t!="") printf("Temperature_Celsius=%s\n",t)}
      }
      if($0~/Power_On_Hours/)          print "Power_On_Hours=" $NF;
      if($0~/Power_Cycle_Count/)       print "Power_Cycle_Count=" $NF;
      if($0~/Reallocated_Sector_Ct/)   print "Reallocated_Sector_Ct=" $NF;
      if($0~/Reported_Uncorrect/)      print "Reported_Uncorrect=" $NF;
      if($0~/Program_Fail_Count/)      print "Program_Fail_Count=" $NF;
      if($0~/Erase_Fail_Count/)        print "Erase_Fail_Count=" $NF;
      if($0~/End-to-End_Error/)        print "End-to-End_Error=" $NF;
      if($0~/UDMA_CRC_Error_Count/)    print "UDMA_CRC_Error_Count=" $NF;
      if($0~/CRC_Error_Count/)         print "CRC_Error_Count=" $NF;
      if($0~/Runtime_Bad_Block/)       print "Runtime_Bad_Block=" $NF;
      if($0~/Total_Bad_Blocks/)        print "Total_Bad_Blocks=" $NF;
      if($0~/Media_Wearout_Indicator/) print "Media_Wearout_Indicator=" $NF;
      if($0~/Percentage_Used/)         print "Percentage_Used=" $NF;
    }' "$1" | sort -u
}

prompt_yes_default(){ local msg="$1" ans; read -r -p "$msg [Y/n] " ans || true; [[ -z "$ans" || "$ans" =~ ^[yY]$ ]]; }
kill_gracefully(){ local t="$1"; kill -INT "$t" 2>/dev/null || true; sleep 0.5; kill -TERM "$t" 2>/dev/null || true; sleep 0.5; kill -KILL "$t" 2>/dev/null || true; }

confirm_proceed(){
  echo "############################################################"
  echo " SMART (baseline) for $DEVICE"
  echo "############################################################"
  local snap; snap="$(smart_snapshot baseline || true)"
  if [[ -n "${snap:-}" && -s "$snap" ]]; then
    sed -n '1,40p' "$snap" || true
  else
    echo "(SMART baseline not available; proceeding without preview)"
  fi
  echo "..."
  read -r -p "Proceed with DESTRUCTIVE stability soak on $DEVICE ? [y/N] " ans
  [[ "${ans:-N}" =~ ^[yY]$ ]]
}

summarize(){
  echo "==================== SUMMARY ($(ts)) ====================" | tee "$SUMMARY"
  echo -e "\n[FIO AGGREGATES]\n" | tee -a "$SUMMARY"
  awk -v OFS='\t' '
    function toMiB(s,  x){
      x=s; gsub(/[^0-9\.A-Za-z\/]/,"",x)
      if (x ~ /KiB\/s$/){sub(/KiB\/s/,"",x); return x/1024}
      if (x ~ /MiB\/s$/){sub(/MiB\/s/,"",x); return x+0}
      if (x ~ /GiB\/s$/){sub(/GiB\/s/,"",x); return (x+0)*1024}
      return 0
    }
    # allow leading spaces before READ:/WRITE:
    /^[[:space:]]*READ:/  { m=$0; sub(/^.*bw=/,"",m); sub(/ .*/,"",m); bw["READ"]+=toMiB(m); c["READ"]++ }
    /^[[:space:]]*WRITE:/ { m=$0; sub(/^.*bw=/,"",m); sub(/ .*/,"",m); bw["WRITE"]+=toMiB(m); c["WRITE"]++ }
    END{
      printf("READ  avg BW: %.2f MiB/s over %d samples\n", (c["READ"]?bw["READ"]/c["READ"]:0),  c["READ"]);
      printf("WRITE avg BW: %.2f MiB/s over %d samples\n", (c["WRITE"]?bw["WRITE"]/c["WRITE"]:0), c["WRITE"]);
    }
  ' "$FIO_LOG" | tee -a "$SUMMARY"

  echo -e "\n[SMART DELTAS]\n" | tee -a "$SUMMARY"
  local base last
  base="$(ls -1 "$SMART_DIR"/smart_*.txt 2>/dev/null | head -n1 || true)"
  last="$(ls -1 "$SMART_DIR"/smart_*.txt 2>/dev/null | tail -n1 || true)"
  if [[ -f "$base" && -f "$last" ]]; then
    declare -A A B
    while IFS='=' read -r k v; do [[ -n "${k:-}" ]] && A["$k"]="$v"; done < <(smart_parse_raw "$base")
    while IFS='=' read -r k v; do [[ -n "${k:-}" ]] && B["$k"]="$v"; done < <(smart_parse_raw "$last")
    for key in "${SMART_KEYS[@]}"; do
      av="${A[$key]:-NA}"; bv="${B[$key]:-NA}"
      if [[ "$av" != "NA" || "$bv" != "NA" ]]; then
        if [[ "$av" =~ ^[0-9]+$ && "$bv" =~ ^[0-9]+$ ]]; then
          printf "%-26s  start=%-8s  end=%-8s  delta=%-8d\n" "$key" "$av" "$bv" "$((bv-av))" | tee -a "$SUMMARY"
        else
          printf "%-26s  start=%-8s  end=%-8s\n" "$key" "$av" "$bv" | tee -a "$SUMMARY"
        fi
      fi
    done
  else
    echo "No SMART snapshots to diff." | tee -a "$SUMMARY"
  fi

  echo -e "\n[KERNEL ERROR EVENTS]\n" | tee -a "$SUMMARY"
  if [[ -s "$ERR_LOG" ]]; then
    echo "Total error lines: $(wc -l < "$ERR_LOG")" | tee -a "$SUMMARY"
    echo -e "\nTop error patterns:\n" | tee -a "$SUMMARY"
    sed -E -e 's/(ata)[0-9]+/\1X/g' -e 's/port [0-9]+/port N/g' \
           -e 's/device [0-9]+:[0-9]+:[0-9]+:[0-9]+/device X:Y:Z:W/g' \
           -e 's/usb [0-9]+-[0-9]+(\.[0-9]+)*/usb BUS-PORT/g' "$ERR_LOG" \
      | awk '{s=$0; gsub(/[0-9]+/,"",s); print s}' \
      | sort | uniq -c | sort -nr | head -50 | tee -a "$SUMMARY"
    echo -e "\nLast 20 error log lines:\n" | tee -a "$SUMMARY"
    tail -n 20 "$ERR_LOG" | tee -a "$SUMMARY"
  else
    echo "No error patterns matched." | tee -a "$SUMMARY"
  fi

  echo -e "\n[LOG FILES]\n$LOG_DIR" | tee -a "$SUMMARY"
  echo "==========================================================" | tee -a "$SUMMARY"

  # --- release singleton lock after summary is printed ---
  { flock -u 9; exec 9>&-; rm -f "$LOCK_FILE"; } 2>/dev/null || true
}

# cleanup/traps MUST be defined before any long-running step
CLEANUP_ONCE=0
FIO_PGID=""; FIO_PID=""; KLOG_PID=""; REQUESTED_STOP=0

cleanup(){
  (( CLEANUP_ONCE == 1 )) && return 0
  CLEANUP_ONCE=1
  REQUESTED_STOP=1
  qecho "== Caught termination; stopping... =="

  # kill fio group if known; also pattern-kill as fallback
  if [[ -n "${FIO_PGID:-}" ]]; then kill_gracefully "-${FIO_PGID}"; fi
  pkill -INT  -f "^fio --name=" 2>/dev/null || true
  pkill -TERM -f "^fio --name=" 2>/dev/null || true
  pkill -KILL -f "^fio --name=" 2>/dev/null || true

  # stop kernel log tail
  if [[ -n "${KLOG_PID:-}" ]] && kill -0 "$KLOG_PID" 2>/dev/null; then kill_gracefully "$KLOG_PID"; fi
  rm -f "$KLOG_PIDFILE"

  # bounded SMART final snapshot
  timeout "$SMART_TIMEOUT" smartctl $SMART_FLAG -a "$DEVICE" >"$SMART_DIR/smart_final_$(date +%F_%H-%M-%S).txt" 2>&1 || true

  summarize
}
trap 'cleanup; exit 0' INT TERM
trap 'cleanup; exit 0' EXIT

#====================== kernel log command =================
if command -v journalctl >/dev/null 2>&1; then KLOG_CMD="journalctl -kf -o short-iso"; else KLOG_CMD="dmesg -w"; fi

#================== startup cleanup prompts ================
if [[ -f "$KLOG_PIDFILE" ]]; then
  OLD_PID="$(cat "$KLOG_PIDFILE" 2>/dev/null || true)"
  if [[ -n "${OLD_PID:-}" && "$OLD_PID" =~ ^[0-9]+$ ]] && kill -0 "$OLD_PID" 2>/dev/null; then
    if prompt_yes_default "Found kernel log tail from a previous run (PID $OLD_PID). Clean it up now?"; then
      kill_gracefully "$OLD_PID"; rm -f "$KLOG_PIDFILE"
    else
      echo "Refusing to start with an existing kernel log tail. Exiting."; exit 1
    fi
  else
    rm -f "$KLOG_PIDFILE"
  fi
fi

if pgrep -f "^fio --name=" >/dev/null 2>&1; then
  if prompt_yes_default "Found running fio jobs from a previous session. Kill them now?"; then
    pkill -INT  -f "^fio --name=" 2>/dev/null || true; sleep 0.5
    pkill -TERM -f "^fio --name=" 2>/dev/null || true; sleep 0.5
    pkill -KILL -f "^fio --name=" 2>/dev/null || true
  else
    echo "Refusing to start while fio jobs are running. Exiting."; exit 1
  fi
fi

#================== SMART baseline & confirm ===============
confirm_proceed || { echo "Aborted."; exit 1; }
qecho "# Stability loop on $DEVICE (started $(ts))"

#================== start kernel log capture ===============
if [[ "${QUIET}" == "1" ]]; then
  stdbuf -oL bash -c "$KLOG_CMD" | stdbuf -oL grep -Eai "$ERROR_PAT" >> "$ERR_LOG" &
else
  stdbuf -oL bash -c "$KLOG_CMD" | tee -a "$KERN_LOG" | stdbuf -oL grep -Eai "$ERROR_PAT" >> "$ERR_LOG" &
fi
KLOG_PID=$!
echo "$KLOG_PID" > "$KLOG_PIDFILE"

#================== fio phase runner ======================
run_fio_phase(){
  local name="$1"; shift
  qbegin "$name"
  setsid bash -c "taskset -c ${PIN_FIO_CORE} fio --name='$name' --filename='$DEVICE' \
      --ioengine='$ENGINE' --direct=1 --group_reporting --randrepeat=0 \
      --status-interval=2 --exitall --exitall_on_error $*" >>"$FIO_LOG" 2>&1 &
  FIO_PID=$!
  # try to capture PGID (fallback killer handles if this fails)
  for _ in {1..10}; do
    FIO_PGID="$(ps -o pgid= -p "$FIO_PID" 2>/dev/null | tr -d ' ' || true)"
    [[ -n "$FIO_PGID" ]] && break
    sleep 0.1
  done
  wait "$FIO_PID" 2>/dev/null || true
  FIO_PID=""; FIO_PGID=""
  qend "$name"
}

#================== main endless loop =====================
ITER=0
while :; do
  ITER=$((ITER+1))
  qecho "$(ts) :: ===== CYCLE $ITER ====="
  smart_snapshot "cycle${ITER}_start_$(date +%F_%H-%M-%S)" >/dev/null

  run_fio_phase "seq_write_${ITER}" --rw=write --bs="$BS_SEQ" \
    --iodepth="$IODEPTH_SEQ" --numjobs=1 --runtime="$SEQ_DURATION" --time_based

  run_fio_phase "seq_read_${ITER}" --rw=read --bs="$BS_SEQ" \
    --iodepth="$IODEPTH_SEQ" --numjobs=1 --runtime="$SEQ_DURATION" --time_based

  run_fio_phase "randrw4k_${ITER}" --rw=randrw --rwmixread="$RW_MIX_READ" --bs="$BS_RAND" \
    --iodepth="$IODEPTH_RAND" --numjobs="$RAND_NUMJOBS" --runtime="$RAND_DURATION" --time_based

  smart_snapshot "cycle${ITER}_end_$(date +%F_%H-%M-%S)" >/dev/null
done

This is an experimental fio stress test to saturate the SATA link on my NanoPi SATA Hat using a brand new ssd to test the stability of the Bridge under recent Mainline vs. FriendlyElecs Vendor Kernel. Running this manually I already observed half the SYS/CPU saturation on mainline but I read about some serious issues and have encountered past issues with Armbian stability myself so before hooking into it (and pinning everything) I really want to see if its REALLY stable right now - or not.

updated with proper trapping, pid, lock file and cleanup on restart ... will make sure that there's just one instance is running.