sonm-fan-control

#!/usr/bin/env bash

VERSION="0.3.2"

cleanup() {
    if [[ $FAIL_TO_START -eq 0 ]]; then
	    echo "INFO: Exit. Transfer fan control to driver."
	    for ((i=0; i<$CARDS_NUM; i++)); do
			nvidia-settings -a [gpu:$i]/GPUFanControlState=0
			if ! [[ $? -eq 0 ]]; then
				echo "CRITICAL: Failed to transfer fan control to the driver. Forcing reboot.."
				reboot
			fi
			if [[ $MANAGE_PL -eq 1 ]]; then
				echo "INFO: Adjusting PL for GPU${i} to default value: ${pl_current[$i]} -> ${pl_defaults[$i]}"
				nvidia-smi -i $i -pl ${pl_defaults[$i]} &>1
			fi
		done
		nvidia-smi -pm 0
	else
		echo "INFO: Exit. Fan-control failed to start."
	fi
}

init() {
	FAIL_TO_START=0
	echo "Sonm-fan-control service, version ${VERSION}"
	if [[ "$(id -u)" != "0" ]]; then
	   echo "ERROR: This script must be run as superuser"
	   FAIL_TO_START=1
	   exit 1
	fi

	if ! lspci -nm | grep '"\(0300\|0302\)" "10de"' 2>&1 >/dev/null; then
	    echo "ERROR: No NVIDIA GPUs detected"
	    FAIL_TO_START=1
	    exit 0
	fi

	if ! [ -f /usr/bin/nvidia-settings ]; then
		echo "ERROR: Nvidia-settings not installed, exit.."
		FAIL_TO_START=1
		exit 1
	fi

	export DISPLAY=:0
	reload_config
	CARDS_NUM=`nvidia-smi -L | grep UUID | wc -l`
	DELAY=5

	echo "INFO: Found ${CARDS_NUM} GPU(s)"
	echo $(print_config)

	for ((i=0; i<$CARDS_NUM; i++)); do
		nvidia-settings -a [gpu:$i]/GPUFanControlState=1 1>/dev/null
		if [ "$?" -ne 0 ]; then
			echo "ERROR: Cannot take control of the GPU fans. Exit"
			exit 1
		fi
	done

	echo "INFO: GPUFanControlState set to 1 for all cards"

	if [[ $MANAGE_PL -eq 1 ]]; then
		echo "INFO: PL management is ENABLED in config"
		enable_pm
		set_pl_defaults
		set_init_pl
	else
		echo "INFO: PL management is DISABLED in config"
	fi
}

print_config() {
	echo "INFO: Fan control settings: TEMP: Min ${MIN_TEMP}°C, Max ${MAX_TEMP}°C, Critical ${CRIT_TEMP}°C; Min fan speed ${MIN_FAN_SPEED}% : "
	echo "PL MANAGEMENT: ${MANAGE_PL} (1-on, 0-off), MaxPL ${MAX_PL}%, MinPL ${MIN_PL}%; PL change step ${PL_CHANGE_STEP}W"
}

reload_config() {
	MANAGE_PL_OLD=$MANAGE_PL
	export $(cat /etc/sonm/fan-control.cfg | grep -v "#")
	if [[ -z $SETTINGS_TS ]]; then
		SETTINGS_TS=$(date +%s)
	fi
	CONFIG_CHANGE_TS=$(stat /etc/sonm/fan-control.cfg --format='%Y')
	if [[ $CONFIG_CHANGE_TS -gt $SETTINGS_TS ]]; then
		echo "INFO: Detected config change"
		echo $(print_config)
		SETTINGS_TS=$(date +%s)
		switch_pl
	fi

	# check config
	if [[ $MIN_TEMP -ge $MAX_TEMP ]]; then
		echo "ERROR: Configuration error, MIN_TEMP>=MAX_TEMP"
		exit 1
	elif [[ $MAX_TEMP -ge $CRIT_TEMP ]]; then
		echo "ERROR: Configuration error, MAX_TEMP>=CRIT_TEMP"
		exit 1
	elif [[ $MIN_FAN_SPEED -lt 0 ]]; then
		echo "ERROR: Configuration error, MIN_FAN_SPEED<=0"
		exit 1
	elif [[ $MIN_FAN_SPEED -gt 100 ]]; then
		echo "ERROR: Configuration error, MIN_FAN_SPEED>=100"
		exit 1
	fi

	return 0
}

switch_pl() {
	set_pl_defaults
	if [[ $(( $MANAGE_PL - $MANAGE_PL_OLD )) -gt 0 ]]; then
		echo "INFO: PL management is ENABLED in config"
		enable_pm
		set_init_pl
	elif [[ $(( $MANAGE_PL - $MANAGE_PL_OLD )) -lt 0 ]]; then
		echo "INFO: PL management is DISABLED in config"
		disable_pm
	fi
}

get_gpu_temp() {
		temp=$(nvidia-smi -i $1 --query-gpu=temperature.gpu --format=csv,noheader)
		ERR=$?

		if ! [[ $ERR -eq 0 ]]; then
			echo "INFO:" $(nvidia-smi -L)
			echo "CRITICAL: Cannot get temp for GPU${1}. Seems like GPU is lost. Forcing reboot"
			reboot
		fi

		if [[ $temp -gt $CRIT_TEMP ]]; then
			echo "CRITICAL: GPU${1} exceeded critical temp. Forcing reboot"
			reboot
		fi

		echo $temp
}

set_pl_defaults() {
	# TODO: Handle errors
	for ((i=0; i<$CARDS_NUM; i++)); do
		pl_defaults[$i]=$(nvidia-smi --format=csv,noheader -i $i --query-gpu=power.default_limit | cut -d "." -f1)
		pl_min[$i]=$(( pl_defaults[$i] * $MIN_PL / 100 ))
		pl_max[$i]=$(( pl_defaults[$i] * $MAX_PL / 100 ))
		echo "INFO: Default (driver settings) PL for GPU${i} is ${pl_defaults[$i]}"
		echo "INFO: PL range for GPU${i}:  ${pl_min[$i]} - ${pl_max[$i]}"
	done
}

set_init_pl() {
	for ((i=0; i<$CARDS_NUM; i++)); do
		nvidia-smi -i $i -pl ${pl_max[$i]} &>1
		pl_current[$i]=${pl_max[$i]}
		# TODO: Handle errors
		echo "INFO: Initial PL for GPU${i} adjusted to ${pl_max[$i]}"
	done
}

enable_pm() {
	echo "INFO: Enabling persistence mode.."
	nvidia-smi -pm 1 &>1
}

disable_pm() {
	echo "INFO: Switching to default PL settings.."
	for ((i=0; i<$CARDS_NUM; i++)); do
		nvidia-smi -i $i -pl ${pl_defaults[$i]} &>1
		# TODO: Handle errors
		echo "INFO: PL for GPU${i} adjusted to ${pl_defaults[$i]}"
	done
	echo "INFO: Disabling persistence mode.."
	nvidia-smi -pm 0 &>1
}

adjust_pl() {
	# TODO: Handle errors
	new_pl=0

	if [[ $2 -gt $(( $MAX_TEMP + 5 )) ]]; then
		new_pl=$(( ${pl_current[$1]} - $PL_CHANGE_STEP ))
		if [[ $new_pl -lt ${pl_min[$1]} ]]; then
			new_pl=${pl_min[$1]}
		fi

	elif [[ $2 -lt $MAX_TEMP ]]; then
		new_pl=$(( ${pl_current[$1]} + $PL_CHANGE_STEP ))
		if [[ $new_pl -gt ${pl_max[$1]} ]]; then
			new_pl=${pl_max[$1]}
		fi

	else
		new_pl=${pl_current[$1]}
	fi

	if ! [[ $new_pl -eq  ${pl_current[$1]} ]]; then
		echo "INFO: Adjusting PL for GPU${i}: ${pl_current[$i]} -> ${new_pl}"
		pl_current[$1]=$new_pl
		nvidia-smi -i $1 -pl $new_pl &>1
	fi
}

if [[ -z $1 ]]; then

	# Executes cleanup function at script exit.
	trap cleanup EXIT

	init

	while true; do
		for ((i=0; i<$CARDS_NUM; i++)); do
			GPU_TEMP=$(get_gpu_temp $i)

			if [[ $GPU_TEMP -lt $MIN_TEMP ]]; then
				FAN_SPEED=$MIN_FAN_SPEED

			elif [[ $GPU_TEMP -gt $MAX_TEMP ]]; then
				echo "WARN: GPU${i} temp ${GPU_TEMP}°C"
				FAN_SPEED=100

			else
				FAN_SPEED=$(( $MIN_FAN_SPEED + ($GPU_TEMP - $MIN_TEMP)*(100 - $MIN_FAN_SPEED)/($MAX_TEMP - $MIN_TEMP) ))
			fi

			echo "INFO: GPU${i} ${GPU_TEMP}°C, fan -> ${FAN_SPEED}%"
			nvidia-settings -a [fan:$i]/GPUTargetFanSpeed=$FAN_SPEED 1>/dev/null

			if [[ $MANAGE_PL -eq 1 ]]; then
				adjust_pl $i $GPU_TEMP
			fi
		done

		sleep $DELAY
		reload_config
	done
fi

if [[ $1 == "version" ]]; then
	echo "Sonm-fan-control service, version ${VERSION}"
	export $(cat /etc/sonm/fan-control.cfg | grep -v "#")
	print_config
else
	echo "Unknown argument"
fi