Running a development version of FORM. #bin #sh #launcher #dev #form

x

#!/bin/bash
#
# Usage:
#   ./x [update|clean|manual|devref|doxygen]
#       [4.1.0|4.2.0|4.2.1|4.3.0|4.3.1]
#       [check|time [-n <times>]]
#       [coverage|gprof|valgrind [<valgrind options>]|helgrind|drd|sanitize]
#       [form|vorm|tform|tvorm|parform|parvorm]
#       [-np <nprocs>]
#       [args...]
#
#   valgrind-options =
#       [--leak-check=<no|summary|yes|full>]
#       [--show-leak-kinds=<set>]
#
set -euo pipefail

TRAP_ERR=false

# Trap ERR to print the stack trace when a command fails.
# See: https://gist.github.com/ahendrix/7030300
errexit() {
  local err=$?
  set +o xtrace
  local code="${1:-1}"
  echo "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}: '${BASH_COMMAND}' exited with status $err" >&2
  # Print out the stack trace described by $FUNCNAME
  if [ ${#FUNCNAME[@]} -gt 2 ]; then
    echo "Traceback:" >&2
    for ((i=1;i<${#FUNCNAME[@]}-1;i++)); do
      echo "  [$i]: at ${BASH_SOURCE[$i+1]}:${BASH_LINENO[$i]} in function ${FUNCNAME[$i]}" >&2
    done
  fi
  echo "Exiting with status ${code}" >&2
  exit "${code}"
}
if $TRAP_ERR; then
  trap 'errexit' ERR
  set -o errtrace
fi

SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
SCRIPT_FILE="$SCRIPT_DIR/$(basename -- "${BASH_SOURCE[0]}")"

SOURCE_DIR=$SCRIPT_DIR
BUILD_DIR="$SCRIPT_DIR/build"

TAG=
VERSION=
TARGET=default
CONFIG_OPTS=()
MAKE_OPTS=()
VALGRIND_OPTS=()
MPIRUN_OPTS=()

# The default tools.
CC=gcc
CXX=g++
LDFLAGS=
MPICC=
MPICXX=
MPIRUN=mpirun
GCOV=gcov
GMP_DIR=
MPFR_DIR=
FLINT_DIR=
ZLIB_DIR=
ZSTD_DIR=
VALGRIND=valgrind
LCOV=lcov
GENHTML=genhtml
GPROF=gprof
GPROF2DOT=gprof2dot
CURL=curl
NPROC=nproc
TIME=/usr/bin/time  # GNU Time

if command -v brew >/dev/null; then
  # We assume that they are installed by Homebrew on Linux, i.e.,
  #   brew install flint gcc@15 gprof2dot lcov valgrind
  #   yes | perl -MCPAN -e 'install Date::Parse'
  #   yes | perl -MCPAN -e 'install JSON::XS'  # optional
  GCC_VERSION=15
  GCC_SUFFIX=-$GCC_VERSION
  CC="$(brew --prefix gcc@$GCC_VERSION)/bin/gcc$GCC_SUFFIX -B$(brew --prefix binutils)/bin"
  CXX="$(brew --prefix gcc@$GCC_VERSION)/bin/g++$GCC_SUFFIX -B$(brew --prefix binutils)/bin"
  GCOV="$(brew --prefix gcc@$GCC_VERSION)/bin/gcov$GCC_SUFFIX"
  GMP_DIR=$(brew --prefix gmp)
  MPFR_DIR=$(brew --prefix mpfr)
  FLINT_DIR=$(brew --prefix flint)
  ZLIB_DIR=$(brew --prefix zlib)
  ZSTD_DIR=$(brew --prefix zstd)
  VALGRIND=$(brew --prefix valgrind)/bin/valgrind
  LCOV=$(brew --prefix lcov)/bin/lcov
  GENHTML=$(brew --prefix lcov)/bin/genhtml
  GPROF=$(brew --prefix binutils)/bin/gprof
  GPROF2DOT=$(brew --prefix gprof2dot)/bin/gprof2dot
  DOT=$(brew --prefix graphviz)/bin/dot
  CURL=$(brew --prefix curl)/bin/curl
fi

if [ -f "$SCRIPT_DIR/x.local" ]; then
  # shellcheck disable=SC1091
  . "$SCRIPT_DIR/x.local"  # local settings
fi

CONFIG_OPTS+=("CC=$CC" "CXX=$CXX")
if [ -n "$LDFLAGS" ]; then
  CONFIG_OPTS+=("LDFLAGS=$LDFLAGS")
fi
CONFIG_OPTS+=(--enable-debug)
if [ -n "$GMP_DIR" ]; then
  CONFIG_OPTS+=("--with-gmp=$GMP_DIR")
fi
if [ -n "$MPFR_DIR" ]; then
  CONFIG_OPTS+=("--with-mpfr=$MPFR_DIR")
fi
if [ -n "$FLINT_DIR" ]; then
  CONFIG_OPTS+=("--with-flint=$FLINT_DIR")
fi
if [ -n "$ZLIB_DIR" ]; then
  CONFIG_OPTS+=("--with-zlib=$ZLIB_DIR")
fi
if [ -n "$ZSTD_DIR" ]; then
  CONFIG_OPTS+=("--with-zstd=$ZSTD_DIR")
fi

DO_CLEAN=false
DO_MANUAL=false
DO_DEVREF=false
DO_DOXYGEN=false
DO_CHECK=false
DO_TIME=false
DO_COVERAGE=false
DO_GPROF=false
DO_VALGRIND=false
DO_HELGRIND=false
DO_DRD=false
DO_SANITIZE=false
N=default

# Parse commend-line arguments.
while (( $# > 0 )); do
  case "$1" in
    form|tform|vorm|tvorm|parform|parvorm)
      TARGET="$1"
      shift
      ;;
    check)
      DO_CHECK=:
      shift
      ;;
    time)
      DO_TIME=:
      shift
      ;;
    -n)
      shift
      if (( $# == 0 )); then
        echo "error: -n option requires a non-zero integer argument" >&2
        exit 1
      fi
      N=$1
      shift
      ;;
    -np)
      shift
      if (( $# == 0 )); then
        echo "error: -np option requires a non-zero integer argument" >&2
        exit 1
      fi
      MPIRUN_OPTS+=(-np "$1")
      shift
      ;;
    cov|coverage)
      DO_COVERAGE=:
      shift
      ;;
    prof|gprof)
      DO_GPROF=:
      shift
      ;;
    valgrind)
      DO_VALGRIND=:
      shift
      ;;
    --leak-check=*|--show-leak-kinds=*)
      VALGRIND_OPTS+=("$1")
      shift
      ;;
    helgrind)
      DO_HELGRIND=:
      shift
      ;;
    drd)
      DO_DRD=:
      shift
      ;;
    sanitize)
      DO_SANITIZE=:
      shift
      ;;
    4.2.0|4.2.1|4.3.0|4.3.1)
      TAG=v$1
      VERSION=$1
      shift
      ;;
    4.1.0)
      TAG=v4.1-20131025
      VERSION=4.1.0
      MAKE_OPTS+=("DATE='Oct 25 2013'")
      shift
      ;;
#   4.0.1)
#     TAG=v4.0-20120410
#     VERSION=4.0.1
#     MAKE_OPTS+=("DATE='Apr 10 2012'")
#     shift
#     ;;
#   4.0.0)
#     TAG=v4.0-20120330
#     VERSION=4.0.0
#     MAKE_OPTS+=("DATE='Mar 29 2012'")
#     shift
#     ;;
    update)
      download_file="$SCRIPT_FILE.tmp$$"
      "$CURL" -o "$download_file" https://gist.githubusercontent.com/tueda/d802b76760451a571d53a974db20d096/raw/x
      if diff -q "$SCRIPT_FILE" "$download_file" >/dev/null; then
        echo "$SCRIPT_FILE up-to-date"
        rm -f "$download_file"
      else
        diff -u "$SCRIPT_FILE" "$download_file" && :
        chmod +x "$download_file"
        mv "$download_file" "$SCRIPT_FILE"
        echo "$SCRIPT_FILE updated"
      fi
      exit
      ;;
    clean)
      DO_CLEAN=:
      shift
      ;;
    manual)
      DO_MANUAL=:
      shift
      ;;
    devref)
      DO_DEVREF=:
      shift
      ;;
    doxygen)
      DO_DOXYGEN=:
      shift
      ;;
    *)
      break
      ;;
  esac
done

check_excl_arg_begin() {
  _check_excl_arg_n=0
  _check_excl_arg_s=""
}

check_excl_arg() {
  local _check_excl_arg_tool
  _check_excl_arg_tool="DO_$(echo "$1" | tr '[:lower:]' '[:upper:]')"
  if ${!_check_excl_arg_tool}; then
    ((++_check_excl_arg_n))
    [[ -z "$_check_excl_arg_s" ]] || _check_excl_arg_s+=" and "
    _check_excl_arg_s+="$1"
  fi
}

check_excl_arg_end() {
  if [ $_check_excl_arg_n -gt 1 ]; then
    echo "error: cannot use $_check_excl_arg_s together" >&2
    exit 1
  fi
}

check_excl_arg_begin
check_excl_arg check
check_excl_arg time
check_excl_arg_end

check_excl_arg_begin
check_excl_arg time
check_excl_arg coverage
check_excl_arg gprof
check_excl_arg valgrind
check_excl_arg helgrind
check_excl_arg drd
check_excl_arg sanitize
check_excl_arg_end

run() {
  _run_echo "$@"
  "$@"
}

_run_echo() {
  echo "Running $*"
}

if [ -n "$VERSION" ]; then
  if [ ! -d "$BUILD_DIR" ]; then
    run mkdir -p "$BUILD_DIR"
    echo '*' >"$BUILD_DIR/.gitignore"
  fi
  if [ ! -d "$BUILD_DIR/$VERSION" ]; then
    run git worktree add --detach --force "$BUILD_DIR/$VERSION" "$TAG"
  fi
  SOURCE_DIR="$BUILD_DIR/$VERSION"
  BUILD_DIR="$BUILD_DIR/$VERSION/build"
fi

if $DO_TIME; then
  case $TARGET in
    default)
      TARGET=form
      ;;
  esac
  case $N in
    default)
      ;;
    *)
      if [[ ! $N =~ ^[1-9][0-9]*$ ]]; then
        echo "error: -n option requires a non-zero integer argument: $N" >&2
        exit 1
      fi
      ;;
  esac
  if [ $# -eq 0 ]; then
    echo "error: nothing to measure time" >&2
    exit 1
  fi
else
  case $N in
    default)
      ;;
    *)
      echo "error: -n $N can be used only with time" >&2
      exit 1
      ;;
  esac
fi

if $DO_COVERAGE; then
  case $TARGET in
    form|tform|parform)
      echo "error: use coverage only with debugging versions" >&2
      exit 1
      ;;
  esac
  if ! $DO_CHECK && [ $# -eq 0 ]; then
    echo "error: nothing to check coverage" >&2
    exit 1
  fi
  BUILD_DIR="$BUILD_DIR/coverage"
  CONFIG_OPTS+=(--enable-coverage)
fi

if $DO_GPROF; then
  case $TARGET in
    default)
      TARGET=form
      ;;
    vorm|tvorm|parvorm)
      echo "error: use gprof only with release versions" >&2
      exit 1
      ;;
  esac
  if ! $DO_CHECK && [ $# -eq 0 ]; then
    echo "error: nothing to perform profiling" >&2
    exit 1
  fi
  BUILD_DIR="$BUILD_DIR/gprof"
  CONFIG_OPTS+=(--enable-profile=gprof)
fi

if $DO_HELGRIND; then
  case $TARGET in
    default)
      TARGET=tvorm
      ;;
  esac
  DO_VALGRIND=:
  DO_HELGRIND=false
  VALGRIND_OPTS+=(--tool=helgrind)
fi

if $DO_DRD; then
  case $TARGET in
    default)
      TARGET=tvorm
      ;;
  esac
  DO_VALGRIND=:
  DO_DRD=false
  VALGRIND_OPTS+=(--tool=drd)
fi

if $DO_SANITIZE; then
  case $TARGET in
    form|tform|parform)
      echo "error: use sanitize only with debugging versions" >&2
      exit 1
      ;;
  esac
  BUILD_DIR="$BUILD_DIR/sanitize"
  CONFIG_OPTS+=(--enable-sanitize)
fi

case $TARGET in
  default)
    TARGET=vorm
    ;;
  parform|parvorm)
    BUILD_DIR="$BUILD_DIR/parform"
    CONFIG_OPTS+=("MPICC=$MPICC" "MPICXX=$MPICXX" --enable-parform)
    ;;
esac

analyze_stat() {
  local n=$1
  shift
  stat_version
  echo "$@" | stat_command 'Command being timed'
  stat_header "$n"
  echo "$@" | stat_number 'User time (seconds)'
  echo "$@" | stat_number 'System time (seconds)'
  echo "$@" | stat_percent 'Percent of CPU this job got'
  echo "$@" | stat_time 'Elapsed (wall clock) time (h:mm:ss or m:ss)'
  echo "$@" | stat_number 'Average shared text size (kbytes)'
  echo "$@" | stat_number 'Average unshared data size (kbytes)'
  echo "$@" | stat_number 'Average stack size (kbytes)'
  echo "$@" | stat_number 'Average total size (kbytes)'
  echo "$@" | stat_number 'Maximum resident set size (kbytes)'
  echo "$@" | stat_number 'Average resident set size (kbytes)'
  echo "$@" | stat_number 'Major (requiring I/O) page faults'
  echo "$@" | stat_number 'Minor (reclaiming a frame) page faults'
  echo "$@" | stat_number 'Voluntary context switches'
  echo "$@" | stat_number 'Involuntary context switches'
  echo "$@" | stat_number 'Swaps'
  echo "$@" | stat_number 'File system inputs'
  echo "$@" | stat_number 'File system outputs'
  echo "$@" | stat_number 'Socket messages sent'
  echo "$@" | stat_number 'Socket messages received'
  echo "$@" | stat_number 'Signals delivered'
  echo "$@" | stat_number 'Page size (bytes)'
  echo "$@" | stat_status 'Exit status'
}

stat_version() {
  echo "  Revision: $("$SCRIPT_DIR/scripts/git-version-gen.sh" -C "$SCRIPT_DIR" | sed -n 3p) ($(git -C "$SCRIPT_DIR" rev-parse --abbrev-ref HEAD))"
}

stat_command() {
  printf "  %s: " "$1"
  extract_values "$1" | sed -n '1p'
}

stat_header() {
  printf "  %-40s" "n = $1"
  printf "%14s (%12s) %14s %14s" 'Mean' 'SD' 'Min' 'Max'
  echo ""
}

stat_number() {
  printf "  %-40s" "$1:"
  extract_values "$1" | calc_stat
}

stat_percent() {
  printf "  %-40s" "$1 (%):"
  extract_values "$1" | sed 's/%$//' | calc_stat
}

stat_time() {
  printf "  %-40s" "${1//h:mm:ss or m:ss/seconds}:"
  extract_values "$1" | awk '{
    split($0, arr, ":")
    if (length(arr) == 1) {
      total_seconds = arr[1]
    } else if (length(arr) == 2) {
      total_seconds = arr[1] * 60 + arr[2]
    } else if (length(arr) == 3) {
      total_seconds = arr[1] * 3600 + arr[2] * 60 + arr[3]
    }
    printf("%s\n", total_seconds)
  }' | calc_stat
}

stat_status() {
  printf "  %s:" "$1"
  extract_values "$1" | sort -n | uniq -c | awk '{
    printf $1 >= 2 ? " %s (%s times)" : " %s", $2, $1
  }
  END {
    printf "\n"
  }'
}

strip_whitespace() {
  cat | sed 's/^[[:space:]]*\|[[:space:]]*$//g'
}

extract_values() {
  cat | grep "$1" | sed "s|$1 *: *||" | strip_whitespace
}

calc_stat() {
  cat | awk '{
    if (min > $1) min = $1
    if (max < $1) max = $1
    sum += $1
    sumsq += ($1)^2
    count++
  }
  BEGIN {
    min = 1e99
    max = -1e99
  }
  END {
    mean = sum / count
    var = (sumsq - sum^2 / count) / (count - 1)
    sd = sqrt(var)
    printf "%14.3f (%12.3f) %14.9g %14.9g\n", mean, sd, min, max
  }'
}

(
  run pushd "$SOURCE_DIR"

  if [ -f configure.ac ] && { [ ! -f configure ] || { [ ! -d build-aux ] && [ ! -f config.sub ] ; }; }; then
    run autoreconf -ifv
  fi

  if [ ! -d "$BUILD_DIR" ]; then
    run mkdir -p "$BUILD_DIR"
    if [ ! -f "$BUILD_DIR/.gitignore" ]; then
      echo '*' >"$BUILD_DIR/.gitignore"
    fi
  fi

  if [ ! -f "$BUILD_DIR/Makefile" ]; then
    run pushd "$BUILD_DIR"
    # shellcheck disable=SC2086
    run "$SOURCE_DIR/configure" "${CONFIG_OPTS[@]}"
    run popd
  fi

  if $DO_CLEAN; then
    run make -C "$BUILD_DIR" clean
    exit
  fi

  if $DO_MANUAL; then
    run make -C "$BUILD_DIR/doc/manual" latex2html
    echo "Output: $(ls -l "$BUILD_DIR/doc/manual/manual/index.html")"
    exit
  fi

  if $DO_DEVREF; then
    run make -C "$BUILD_DIR/doc/devref" latex2html
    echo "Output: $(ls -l "$BUILD_DIR/doc/devref/devref/index.html")"
    exit
  fi

  if $DO_DOXYGEN; then
    rm -f "$BUILD_DIR/doc/doxygen/html/index.html"
    run make -C "$BUILD_DIR/doc/doxygen" html
    echo "Output: $(ls -l "$BUILD_DIR/doc/doxygen/html/index.html")"
    exit
  fi

  run make -C "$BUILD_DIR/sources" "$TARGET" -j "$($NPROC)" "${MAKE_OPTS[@]}"
)

TARGET_BIN="$BUILD_DIR/sources/$TARGET"

if $DO_COVERAGE; then
  run "$LCOV" -z -d "$BUILD_DIR"
fi

if $DO_CHECK; then
  if $DO_VALGRIND; then
    if (( ${#VALGRIND_OPTS[@]} > 0 )); then
      run "$SCRIPT_DIR/check/check.rb" --form "$TARGET_BIN" --valgrind "$VALGRIND" --valgrind-opts "${VALGRIND_OPTS[@]}" --stat "$@"
    else
      run "$SCRIPT_DIR/check/check.rb" --form "$TARGET_BIN" --valgrind "$VALGRIND" --stat "$@"
    fi
  else
    run "$SCRIPT_DIR/check/check.rb" --form "$TARGET_BIN" --stat "$@"
  fi
elif $DO_TIME; then
  case $N in
    default|1)
      run "$TIME" -v "$TARGET_BIN" "$@"
      ;;
    *)
      times=()
      for ((i=1; i<=N; i++)); do
        s=$({ run "$TIME" -v "$TARGET_BIN" "$@" 1>&3; } 2>&1)
        times+=("$s"$'\n')
      done 3>&1
      echo >&2
      analyze_stat "$N" "${times[@]}" >&2
      ;;
  esac
else
  if [ $# -ge 1 ]; then
    case $TARGET in
      parform|parvorm)
        if command -v brew >/dev/null; then
          # quick fix for Linuxbrew
          export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$(brew --prefix)/lib"
        fi
        if $DO_VALGRIND; then
          # shellcheck disable=SC2086
          run "$MPIRUN" "${MPIRUN_OPTS[@]}" "$VALGRIND" "${VALGRIND_OPTS[@]}" "$TARGET_BIN" "$@"
        else
          # shellcheck disable=SC2086
          run "$MPIRUN" "${MPIRUN_OPTS[@]}" "$TARGET_BIN" "$@"
        fi
        ;;
      *)
        if $DO_VALGRIND; then
          # shellcheck disable=SC2086
          run "$VALGRIND" "${VALGRIND_OPTS[@]}" "$TARGET_BIN" "$@"
        else
          run "$TARGET_BIN" "$@"
        fi
        ;;
    esac
  fi
fi

if $DO_COVERAGE; then
  run "$LCOV" -c -d "$BUILD_DIR" -o "$BUILD_DIR/coverage.info" --gcov-tool "$GCOV"
  run "$LCOV" -r "$BUILD_DIR/coverage.info" "$GCC_VERSION" -o "$BUILD_DIR/coverage.info" --ignore-errors unused
  run "$LCOV" -r "$BUILD_DIR/coverage.info" "/usr/include/" -o "$BUILD_DIR/coverage.info" --ignore-errors unused
  run "$LCOV" -r "$BUILD_DIR/coverage.info" "/include/bits/" -o "$BUILD_DIR/coverage.info" --ignore-errors unused
  run "$LCOV" -r "$BUILD_DIR/coverage.info" "/.linuxbrew/opt/" -o "$BUILD_DIR/coverage.info" --ignore-errors unused
  run "$LCOV" -r "$BUILD_DIR/coverage.info" "extern/" -o "$BUILD_DIR/coverage.info" --ignore-errors unused
  run "$GENHTML" -o "$BUILD_DIR/html" "$BUILD_DIR/coverage.info"
  echo "Output: $(ls -l "$BUILD_DIR/html/index.html")"
fi

if $DO_GPROF; then
  # NOTE: "run" doesn't work with pipes.
  _run_echo "$GPROF $TARGET_BIN | $GPROF2DOT | $DOT -Tpng -o gprof.png"
  "$GPROF" "$TARGET_BIN" | "$GPROF2DOT" | "$DOT" -Tpng -o gprof.png
  echo "Output: $(ls -l "$(pwd)/gprof.png")"
fi