Dell R630 Fan Manager (bash script, control loop based, needs ipmitool)

ipmi-fanctrl-dell_r630.sh

#!/bin/bash
# Copyright 2024 - 2024, Jaremy Hatler
# SPDX-License-Identifier: MIT

## Dell R630 Fan Control Script

# This script controls the fan speed on a Dell r630 server in response to the CPU temperature.
#
# In some cases (nonstandard firmware, aftermarket PCI cards, etc.), the internal fan controller
# will consistently maintain high RPMs. This script uses IPMI to enable manual fan control and set
# the fan speed across all fans. An exit trap is used to ensure automatic fan control is enabled
# upon exiting. On each iteration, the script logs its data and any changes it makes.
#
# A control loop is used which monitors the CPU package temperatures, inlet temp, and exhaust temp
# and adjusts the fan speeds in response. The user can set the parameters of this control loop in
# the section below, including the loop delay. At a high level, the control loop begins by setting
# the fans to the configured start speed. It will then use the difference between the target and
# actual package temperatures to determine whether to increase or decrease the fan speed, and by
# how much.
#
# If the packages are at or below the target temp, the control loop will reduce the fans speeds
# until an equilibrium is reached. If the inlet temperature is higher than the target, it plus
# the configured hysteresis offset will be used as the target.
#
# If the package alarm temperature is met, the fans will immediately be set to their configured
# maximums until the temperature begins to drop. If the configured critical temperature is met
# for longer than the specified hysteresis wait, the user-specified critical event command will
# be run. The script becomes more aggressive as the exhaust temperature and package temperature
# near the package alarm temperature.

# Proof of concept commands:
#   Get Package 0 Temp: cat /sys/devices/platform/coretemp.0/hwmon/hwmon0/temp1_input
#   Get Package 1 Temp: cat /sys/devices/platform/coretemp.1/hwmon/hwmon1/temp1_input
#   Get Inlet Temp: ipmitool sensor get 'Inlet Temp' | grep Reading | cut -f2 -d:
#   Get Exhaust Temp: ipmitool sensor get 'Exhaust Temp' | grep Reading | cut -f2 -d:
#   Enable manual fan control: ipmitool raw 0x30 0x30 0x01 0x00
#   Disable manual fan control: ipmitool raw 0x30 0x30 0x01 0x01
#   Set Fan Speeds to 0%: ipmitool raw 0x30 0x30 0x02 0xff 0x00
#   Set Fan Speeds to 100%: ipmitool raw 0x30 0x30 0x02 0xff 0x64


## User Parameters

DEBUG=0 # Enable debugging
TARGET_TEMP=35 # The target temperature in Celsius
HYSTERESIS_OFFSET=5 # The hysteresis offset in Celsius
LOOP_DELAY=5 # The delay between each loop iteration in seconds
START_SPEED=30 # The starting fan speed in percent
MIN_SPEED=5 # The minimum fan speed in percent
MAX_SPEED=70 # The maximum fan speed in percent
PACKAGE_ALARM_TEMP=70 # The package alarm temperature in Celsius
CRITICAL_TEMP=85 # The critical temperature in Celsius
HYSTERESIS_WAIT=300 # The hysteresis wait time in seconds
CRITICAL_EVENT_COMMAND="shutdown -h now" # The command to run when the critical temperature is met
HYSTERESIS_MULTIPLIER="2.0" # The multiplier for the exhaust temperature hysteresis

## Exit Trap

function _exit_trap() {
    # Ensure automatic fan control is enabled upon exiting
    _disable_manual_fan_control
}


## Functions

function _pkg_temp() {
    # Returns the average of all the platform core temperatures
    local _acc=0
    local _count=0
    for _core in /sys/devices/platform/coretemp.*; do
        _acc=$(( _acc + $(cat $_core/hwmon/hwmon*/temp1_input) ))
        _count=$(( _count + 1000 ))
    done
    echo $((_acc / _count))
}

function _inlet_temp() {
    # Returns the inlet temperature
    ipmitool sensor get 'Inlet Temp' | grep -F 'Sensor Reading' | cut -f2 -d: | cut -f2 -d' ' 2>/dev/null
}

function _exhaust_temp() {
    # Returns the exhaust temperature
    ipmitool sensor get 'Exhaust Temp' | grep -F 'Sensor Reading' | cut -f2 -d: | cut -f2 -d' ' 2>/dev/null
}

function _set_fan_speed() {
    # Sets the fan speed to the given percentage
    local _speed=$1

    # Ensure the speed is within the valid range
    if [[ $_speed -lt $MIN_SPEED ]]; then
        _speed=$MIN_SPEED
    elif [[ $_speed -gt $MAX_SPEED ]]; then
        _speed=$MAX_SPEED
    fi

    # Convert the speed to hex
    local _hex_speed=$(printf "0x%x" $_speed)

    # Set the fan speed
    ipmitool raw 0x30 0x30 0x02 0xff $_hex_speed >&/dev/null
}

function _enable_manual_fan_control() {
    # Enables manual fan control
    ipmitool raw 0x30 0x30 0x01 0x00 >&/dev/null
}

function _disable_manual_fan_control() {
    # Disables manual fan control
    ipmitool raw 0x30 0x30 0x01 0x01 >&/dev/null
}

function _calculate_fan_speed() {
    # Calculates the fan speed based on the target temperature and the actual temperature
    local _target_temp=$1
    local _pkg_temp=$2
    local _exhaust_temp=$3
    local _fan_speed=$4
    local _start_speed=$4

    # Int Rounding Support
    local _rnd_bgn="scale=8; a=("
    local _rnd_end=" + 0.5); scale=0; a/1"

    # Starting factors
    local _hysteresis_factor="1.0"
    local _speed_factor="0"

    # Calculate the hysteresis temperature and round to the nearest integer
    local _hysteresis_offset=$(echo "$_rnd_bgn($HYSTERESIS_OFFSET * $HYSTERESIS_MULTIPLIER)$_rnd_end" | bc)
    local _hysteresis_temp=$(echo "$_rnd_bgn($PACKAGE_ALARM_TEMP - $_hysteresis_offset)$_rnd_end" | bc)

    # Calculate the difference between the actual temperature and target
    local _temp_diff=$(( _pkg_temp - _target_temp ))
    local _temp_diff_abs=${_temp_diff#-}

    # Offset speed factor based on the exhaust temperature
    if [[ $_exhaust_temp -gt $_hysteresis_temp ]]; then
        _speed_factor=$(echo "$_speed_factor + 0.75" | bc -l)
    fi

    # Calculate the fan speed change based on the range of the temperature difference
    if [[ $_temp_diff_abs -gt 32 ]]; then
        _speed_factor="2.0"
    elif [[ $_temp_diff_abs -gt 16 ]]; then
        _speed_factor="1.375"
    elif [[ $_temp_diff_abs -gt 8 ]]; then
        _speed_factor="0.875"
    elif [[ $_temp_diff_abs -gt 4 ]]; then
        _speed_factor="0.5"
    elif [[ $_temp_diff_abs -gt 2 ]]; then
        _speed_factor="0.25"
    fi

    # Update hysteresis factor based on the package temperature compared to the hysteresis temperature
    if [[ $_pkg_temp -gt $_hysteresis_temp ]]; then
        _hysteresis_factor=$(echo "$_hysteresis_factor + 2" | bc)
    else
        _hysteresis_factor=$(echo "$_hysteresis_factor + 1" | bc)
    fi

    # Calculate the speed change based on the speed factor and hysteresis factor, ensure the range
    local _speed_change=$(echo "$_rnd_bgn($_speed_factor * $_hysteresis_factor)$_rnd_end" | bc)
    if [[ $_speed_change -gt 5 ]]; then
        _speed_change=5
    fi

    # Give the speed change a direction
    if [[ $_temp_diff -lt 0 ]]; then
        _speed_change=$(( _speed_change * -1 ))
    fi

    # Ensure the fan speed is reduced if the package temperature at or below the target
    if [[ $_temp_diff -le 0 ]] && [[ $_speed_change -eq 0 ]]; then
        _speed_change=-1
    fi

    # Calculate the new fan speed
    _fan_speed=$(( _fan_speed + _speed_change ))
    if [[ $_fan_speed -lt 1 ]]; then
        _fan_speed=1
    elif [[ $_fan_speed -gt 99 ]]; then
        _fan_speed=99
    fi

    # Debugging
    if [[ "$DEBUG" != "0" ]]; then
        echo "Start Speed: $_start_speed"
        echo "Package Temp: $_pkg_temp"
        echo "Exhaust Temp: $_exhaust_temp"
        echo "Target Temp: $_target_temp"
        echo "Hysteresis Temp: $_hysteresis_temp"
        echo "Temp Diff: $_temp_diff"
        echo "Speed Factor: $_speed_factor"
        echo "Hysteresis Factor: $_hysteresis_factor"
        echo "Fan Speed: $_fan_speed"
    fi >&2

    echo $_fan_speed
}

function _log_data() {
    # Logs the data to a file
    local _pkg_temp=$1
    local _inlet_temp=$2
    local _exhaust_temp=$3
    local _target_temp=$4
    local _fan_speed=$5

    # Log the data
    local _msg="$(date +%s) $(date) - "

    _msg+="Package Temp: $_pkg_temp, "
    _msg+="Inlet Temp: $_inlet_temp, "
    _msg+="Exhaust Temp: $_exhaust_temp, "
    _msg+="Target Temp: $_target_temp, "
    _msg+="Fan Speed: $_fan_speed [$MIN_SPEED - $MAX_SPEED]"

    echo "$_msg" | tee -a /var/log/fanctrl.log >&2
}

function _run_critical_event() {
    # Runs the critical event command
    eval "$CRITICAL_EVENT_COMMAND"
}

## Main
function _main() {
    local critical_event_wait=0 # The time since the critical temperature was met
    local fan_speed=$START_SPEED # The current fan speed
    local new_fan_speed=$START_SPEED # The new fan speed

    # Enable manual fan control and set the initial fan speed
    _enable_manual_fan_control
    _set_fan_speed $fan_speed

    # Main control loop
    while true; do
        # Get the current temperatures
        local pkg_temp=$(_pkg_temp)
        local inlet_temp=$(_inlet_temp)
        local exhaust_temp=$(_exhaust_temp)

        # Calculate the target temperature
        local target_temp=$TARGET_TEMP
        if [[ $inlet_temp -gt $target_temp ]]; then
            target_temp=$((inlet_temp + HYSTERESIS_OFFSET))
        fi

        # Debugging
        if [[ "$DEBUG" != "0" ]]; then
            echo "Start Fan Speed: $fan_speed"
            echo "Crtical Wait: $critical_event_wait"
            echo "Package Temp: $pkg_temp"
            echo "Inlet Temp: $inlet_temp"
            echo "Exhaust Temp: $exhaust_temp"
            echo "Target Temp: $target_temp"
        fi >&2

        # Check for critical temperature
        if [[ $pkg_temp -ge $CRITICAL_TEMP ]]; then
            critical_event_wait=$((critical_event_wait + LOOP_DELAY))
            if [[ $critical_event_wait -ge $HYSTERESIS_WAIT ]]; then
                _run_critical_event
            fi
        else
            critical_event_wait=0
        fi

        # Check for package alarm temperature
        if [[ $pkg_temp -ge $PACKAGE_ALARM_TEMP ]]; then
            new_fan_speed=$MAX_SPEED
        else
            new_fan_speed=$(_calculate_fan_speed $target_temp $pkg_temp $exhaust_temp $fan_speed)
        fi

        # Set the new fan speed if it has changed
        if [[ $new_fan_speed -ne $fan_speed ]]; then
            fan_speed=$new_fan_speed
            _set_fan_speed $fan_speed
        fi

        # Log the data
        _log_data $pkg_temp $inlet_temp $exhaust_temp $target_temp $fan_speed

        # Sleep for the loop delay
        sleep $LOOP_DELAY
    done
}

if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
    trap _exit_trap EXIT
    _main
fi