varnishgather

#!/bin/sh
#
# varnishgather - a data collection script for debugging Varnish issues.
#
# This script is non-invasive and can be run on production Varnish
# servers with no ill effect.
#
# Copyright (C) 2010-2016 Varnish Software AS <support@varnish-software.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

export LC_ALL=C


# Defines

HOSTPORT=""
NAME=""
VARNISH=""
VARNISHADMARG=""
SECRET=""
STATCMD=""
ITEM=0
TOPDIR=$(mktemp -d ${TMPDIR:-/tmp}/varnishgather.XXXXXXXX)
ID="$(cat /etc/hostname)-$(date +'%Y%m%d-%H%M%S')"
RELDIR="varnishgather-$ID"
ORIGPWD=$PWD
VERSION=1.100
USERID=$(id -u)
PID_ALL_VARNISHD=$(pidof varnishd 2> /dev/null)
PID_ALL_VARNISHD_COMMA=$(pidof varnishd 2> /dev/null | sed 's/ /,/g')
PID_VARNISHD=$(pgrep varnishd 2> /dev/null)
PID_CACHEMAIN=$(pgrep cache-main 2> /dev/null)

# Helpers

# Takes a string, translates it into something we can use as a log
# file name

n_opt() {
	echo "$1" | tr '/ ' __
}

item_num() {
	printf "%.3d" "$ITEM"
}

logname() {
	echo "$@" | tr -c '[:alnum:]-' '[_*]' | sed 's/_*$//'
}

log() {
	echo "$*" >>"$LOG"
}

info() {
	echo "Info: $*" >&2
	log "$@"
}

warn() {
	echo "Warning: $*" >&2
	log "$@"
}

incr() {
	ITEM=$((ITEM + 1))
}

taskecho() {
	echo "Task: $ITEM: $*"
}

banner() {
	log "--------------------------------"
	log "Item $ITEM: $*"
	log "--------------------------------"
	taskecho "$*"
}

run() {
	incr

	command -v "$1" >/dev/null 2>&1 || return 0

	OLDLOG="$LOG"
	LOG="${DIR}/$(item_num)_$(logname "$@")"
	banner "$@"
	("$@") >>"$LOG" 2>&1
	LOG="$OLDLOG"
}

pack_vcls() {
	incr
	taskecho "Compressing VCLs"
	tar cf "$DIR/$(item_num)_vcls.tar" $(vcl_find) >>"$LOG" 2>&1
}

runpipe_recurse() {
	CMD="$1";
	shift;
	if [ "x$*" = "x" ]; then
		${CMD} 2>&1
	else
		${CMD} 2>&1 | runpipe_recurse "$@"
	fi
}

pipeprint() {
	CMD="$1";
	shift;
	if [ "x$*" = "x" ]; then
		echo ${CMD}
	else
		echo ${CMD} "|" $(pipeprint "$@")
	fi
}

runpipe() {
	incr
	OLDLOG="$LOG"
	LOG="${DIR}/$(item_num)_$(logname "$@")"
	banner $(pipeprint "$@")
	runpipe_recurse "$@" >> ${LOG} 2>&1
	LOG="$OLDLOG"
}

mycat() {
	if [ -r $1 ]; then
		run cat $1
	else
		incr
	fi
}

get_pid () {
	pidir="$(ls /proc/ | grep -E '^[0-9]+$' 2>&1)"
	for pid in $pidir; do
		pidnme="$(grep -e "^Name" /proc/$pid/status 2>&1 | awk '{print $2}')"
		if [ $pidnme = "varnishd" ] && [ -z "${PID_VARNISHD}" ]; then
			PID_VARNISHD="${PID_VARNISHD}$(grep -e "^Pid" /proc/$pid/status | awk '{print $2 }')"
		elif [ $pidnme = "cache-main" ] && [ -z "${PID_CACHEMAIN}" ]; then
			PID_CACHEMAIN="${PID_CACHEMAIN}$(grep -e "^Pid" /proc/$pid/status | awk '{print $2 }')"
		fi
	done

	if [ -z "${PID_ALL_VARNISHD}" ] && [ -z "${PID_ALL_VARNISHD_COMMA}" ]; then
		PID_ALL_VARNISHD=$(echo "${PID_VARNISHD} ${PID_CACHEMAIN}" | tr '\n' ' ')
		PID_ALL_VARNISHD_COMMA=$(echo "${PID_VARNISHD} ${PID_CACHEMAIN}" | tr '\n' ' ' | sed 's/ /,/g')
	fi
}

vadmin() {
	if [ ! -z "${VARNISHADMARG}" ]; then
		run varnishadm ${VARNISHADMARG} -- "$@" 2>/dev/null
	fi
}

vadmin_getvcls() {
	if [ ! -z "${VARNISHADMARG}" ]; then
		varnishadm ${VARNISHADMARG} vcl.list 2>/dev/null | awk -F ' *|/' '$2 !~ "label" {print $5}'
	fi
}

vcl_find_path() {
	if [ ! -z "${VARNISHADMARG}" ]; then
		varnishadm ${VARNISHADMARG} param.show vcl_path 2>/dev/null |
		awk '/Value is:/ { print $3 }' |
		tr : '\n'
	fi
	echo /etc/varnish
}

vcl_already_found() {
	cat $TOPDIR/vcl_files.txt $TOPDIR/vcl_includes.txt |
	grep -Fx "$1"
}

vcl_find_includes() {
	vcl=$1

	vcl_already_found "$vcl" ||
	printf '%s\n' "$vcl" >>$TOPDIR/vcl_includes.txt

	sed 's/[;]/;\n/g' 2>/dev/null <"$vcl" |
	awk -F'"' '/\yinclude\s+"[^"]+"\s*;/ { print $2 }' |
	while read vcl_inc
	do
		# TODO: technically, relative includes can fall back to vcl_path
		test -n "${vcl_inc%%/*}" &&
		vcl_inc="$(dirname "$vcl")/$vcl_inc"

		vcl_already_found "$vcl_inc" ||
		vcl_find_includes "$vcl_inc"
	done
}

vcl_find() {
	vcl_find_path |
	sort |
	uniq |
	xargs -I {} find {} -name '*.vcl' -type f >$TOPDIR/vcl_files.txt

	touch $TOPDIR/vcl_includes.txt
	while read vcl
	do
		vcl_find_includes "$vcl"
	done <$TOPDIR/vcl_files.txt

	cat $TOPDIR/vcl_files.txt $TOPDIR/vcl_includes.txt |
	sort |
	uniq

	rm -f $TOPDIR/vcl_files.txt $TOPDIR/vcl_includes.txt
}

findmodsec() {
        base_modsec=$(find /etc/varnish/modsec -name '*' 2>/dev/null)
        [ -z "${base_modsec}" ] && return

        modsec=$(for file in ${base_modsec}; do
                is_absolute=$(echo $file | sed -e '/^\//!d')
                if [ -z "$is_absolute" ]; then
                        file="/etc/varnish/$file"
                fi

		if [ -e "$file" ]; then
			echo $file;
		fi

        done | sort | uniq)
	tar cf "$DIR/$(item_num)_modsec.tar" $modsec >>"$LOG" 2>&1
}
foreacharg()
{
	for pid in ${PID_VARNISHD}
	do
		if [ ! -d "/proc/$pid" ]
		then
			info "Automatic argument detection via proc failed for $pid"
			continue
		fi
		awk -v RS='\\x00' -v opt="$1" -v arg=none '
			arg == "next" { arg = "now" }
			index($0,opt) == 1 { arg = "opt" }
			$0 == opt { arg = "next" }
			arg == "now" {
				print $0
				arg = "none"
			}
			arg == "opt" {
				print substr($0,length(opt)+1)
				arg = "none"
			}
		' "/proc/$pid/cmdline"
	done
}

check_tools() {
	missing=
	for command in varnishadm varnishd varnishlog varnishstat; do
		if [ ! $(command -v $command) ]; then
			missing="$missing$command "
		fi
	done
	if [ -n "$missing" ]; then
		echo "#######################################################"
		echo "Some binaries are missing from PATH, the results might"
		echo "not be complete:"
		echo ""
		echo "    $missing"
		echo ""
		echo "Please amend your PATH and run again."
		echo "#######################################################"
		sleep 10
	fi
}

show_packages() {
	run dpkg --list "$@"
	run dpkg --status "$@"
	run rpm --query --all "$@"
	run rpm --query --all --info "$@"
}

show_limits() {
	file="/proc/$(echo $PID_VARNISHD | awk '{print $1;}')/limits"

	incr

	OLDLOG="$LOG"
	LOG="${DIR}/$(item_num)_limits"
	banner limits
	log "$file"
	cat "$file" >> ${LOG} 2>&1
	LOG="$OLDLOG"
}

blockdev_cat() {
	echo -n >>${LOG} "$1:"
	cat "$1" >> ${LOG} 2>&1
}

call_blockdev() {
	incr

	OLDLOG="$LOG"
	LOG="${DIR}/$(item_num)_blockdev_info_$(logname "$1")"
	banner "blockdev info $1"
	blockdev_cat "$1/device/model"
	blockdev_cat "$1/queue/logical_block_size"
	blockdev_cat "$1/queue/physical_block_size"
	blockdev_cat "$1/queue/read_ahead_kb"
	blockdev_cat "$1/queue/minimum_io_size"
	blockdev_cat "$1/queue/optimal_io_size"
	LOG="$OLDLOG"
}

upload_fail() {
	warn "$1"
	echo "==============================================================================="
	echo "Please submit the file: $TGZ"
	echo "==============================================================================="
}

# Upload gather to filebin using curl
upload() {
	if command -v curl >/dev/null
	then
			FILEBIN="https://filebin.varnish-software.com"
			# Generate BIN name
			BIN=$(head /dev/urandom | tr -dc a-z0-9 | head -c16)-T$UPLOAD
			TGZ="varnishgather.${ID}.tar.gz"
			if test ! -f ${TGZ}
			then
				echo "Error: The file ${TGZ} does not exist. No file to upload."
				return 1
			fi

			URL="${FILEBIN}/${BIN}/${TGZ}"
			CURLSTATUS=$(curl --data-binary "@$TGZ" $URL \
				--header "Varnishgather: ${VERSION}" \
				--progress-bar --silent --output /dev/null \
				--connect-timeout 60 --max-time 1800 \
				--write-out '%{http_code}')
			if [ ! "$CURLSTATUS" -eq "201" ]
			then
				upload_fail "Failed to upload $TGZ to $FILEBIN, http status code: $CURLSTATUS"
				return 0
			fi
			echo "==============================================================================="
			echo "varnishgather $TGZ"
			echo "Uploaded to: $FILEBIN/$BIN"
			echo "==============================================================================="
	else
		upload_fail "\"curl\" binary not found in path, please submit file by uploading to $FILEBIN"
	fi
}

usage() {
	cat <<_EOF_
Usage: $0 [-n name] [-T host:port] [-S secretfile] [-h]

Varnishgather gathers various system information into a single tar-ball.

 -n <name>            Provide the name, same as the -n argument to varnishd
 -T <host:port>       Provide host and port for the management interface
                      Same as the -T option to varnishd.
 -S <secretfile>      Provide a secret file, same as -S to varnishd.
 -u <identifier>      Upload generated varnishgather to filebin to
                      filebin.varnish-software.com, curl needed.
 -p                   Perform a perf capture.
 -h                   Show this text.

All arguments are optional. varnishgather will attempt to detect the
arguments automatically, but will likely be confused if you have multiple
varnish servers running. Using -n should be enough to specify the desired
varnish instance. If not, use -T and -S too.
_EOF_
	exit $1
}

##############################
# Proper execution starts here
##############################

while getopts hpn:S:T:u: opt
do
	case $opt in
	u) UPLOAD=$OPTARG ;;
	p) PERF="TRUE" ;;
	n)
		NAME="-n $OPTARG"
		ID="$ID-$(n_opt "$OPTARG")"
		RELDIR="$RELDIR-$(n_opt "$OPTARG")"
		;;
	T) HOSTPORT="-T $OPTARG" ;;
	S)
		if [ -r "$OPTARG" ]
		then
			SECRET="-S $OPTARG";
		else
			info "Secret file $OPTARG not readable."
			exit 1
		fi
		;;
	h)
		usage 0
		;;
	*)
		usage 1
		;;
	esac
done

shift $((OPTIND - 1))

if [ $# -gt 0 ]
then
	info "Unknown argument: $1"
	usage 1
fi

STATCMD=$NAME
VARNISHADMARG="$NAME $SECRET $HOSTPORT"
VARNISH=$(varnishd -V 2>&1)

# Set up environment

DIR="$TOPDIR/$RELDIR"
LOG="$DIR/varnishgather.log"

mkdir -p "$DIR"
cd "$DIR"

cleanup () {
	rm -rf "$TOPDIR"
	exit 1
}
trap cleanup EXIT 2

info "Varnishgather version: $VERSION"
info "Invoked by: $USERID"
info "Command line: $*"
info "Working directory: $DIR"

if [ "${USERID}" -ne "0" ]; then
	echo "#######################################################"
	echo "Running as non-root, the results might not be complete."
	echo "Please run again as root."
	echo "#######################################################"
	sleep 10
fi

check_tools

if [ ! $(command -v pidof) ] || [ ! $(command -v pgrep) ]; then
	 get_pid
fi

info "Complete varnishadm command line deduced to: $VARNISHADMARG"
run varnishd -V
run varnish-agent -V
run vha-agent -V
run broadcaster -V
run nats-server -v
run vcli version
run varnish-controller-brainz -version
run varnish-controller-agent -version
run varnish-controller-api-gw -version
run varnish-controller-ui -version
run date
run dmesg
run dmesg -T
run hostname
mycat /etc/hostname
mycat /var/log/dmesg
mycat /proc/cpuinfo
mycat /proc/version
runpipe "ps aux" "egrep (varnish|vha-agent|vac|vstatd|vcs|apache|mysql|nginx|httpd|stud|hitch|stunnel|api-engine|broadcaster|mongod|varnish-controller)"
runpipe "netstat -np" "wc -l"
runpipe "netstat -np" "grep ESTABLISHED" "wc -l"
run uptime
run free -m
runpipe "ps axo pid,user,pmem,vsz,rss,exe,comm k -pmem" "head -n 11"
run vmstat -w 1 3
run mpstat -P ALL 1 3
run iostat 1 3
run lsb_release -a
mycat /etc/redhat-release
run sysctl -a
run getenforce
runpipe "semodule -l" "grep varnish"
runpipe "semodule -lfull" "grep varnish"
run ausearch --context varnish
runpipe "ausearch --context varnish --raw" "audit2allow"
runpipe "semanage fcontext -l" "grep varnish"
run semodule_unpackage /usr/share/selinux/targeted/varnish-plus.pp /dev/null /dev/stdout
run umask

show_packages \
	'varnish*' \
	'*vmod*' \
	vac \
	vha-agent \
	hitch \
	gcc \
	mongodb \
	'*jemalloc*' \
	'*pcre*' \
	'*openssl*'\
	'*libssl*'\
	ca-certificates

run systemctl cat \
	varnish \
	varnish-agent \
	varnish-controller-agent \
	varnish-controller-api-gw \
	varnish-controller-nats \
	varnish-controller-ui \
	vcs \
	vcs-agent \
	vstatd \
	vstatdprobe \
	vac

runpipe "systemctl cat varnish-controller-brainz" "grep -v _PASS"

run netstat -s
run ip a
run ip n
run ip r
run ip -s l
run ifconfig
run uname -a
run mount
runpipe mount sort
run df -h
run fdisk -l
run lvdisplay -vm
run vgdisplay -v
run pvdisplay -v
run varnishstat -1 $STATCMD
run varnishstat -j $STATCMD
run ldd "$(command -v varnishd)"
run varnishadm ${VARNISHADMARG} debug.jemalloc_stats
run varnishscoreboard $STATCMD

NETSTAT="/bin/netstat"
run "$NETSTAT" -nlpt
run "$NETSTAT" -np

pack_vcls

mycat /etc/init.d/varnish
mycat /etc/default/varnish

# MSE3 configuration file
for arg in $(IFS="
" foreacharg -s)
do
	MSE3_PATH=$(echo "$arg" | awk -F, '/^mse|=mse/ {print $2}')
	case "$VARNISH" in
	*-6.0.*)
		[ -f "${MSE3_PATH}" ] && mycat "${MSE3_PATH}" ;;
	*) ;;
	esac
done

# -I cli file
for arg in $(IFS="
" foreacharg -I)
do
	mycat "$arg"
done

# -A tls.cfg file
for arg in $(IFS="
" foreacharg -A)
do
	mycat "$arg"
done

mycat /etc/varnish/nats.conf

mycat /etc/sysconfig/varnish
mycat /etc/varnish/varnish.params
mycat /sys/kernel/mm/transparent_hugepage/enabled
mycat /sys/kernel/mm/redhat_transparent_hugepage/enabled
mycat /proc/user_beancounters
mycat /proc/meminfo
mycat /proc/crypto
mycat /proc/sys/net/ipv4/tcp_tw_reuse
for pid in ${PID_VARNISHD}; do
	run cat -v /proc/$pid/cmdline
	mycat /proc/$pid/cgroup
done
show_limits

mycat /etc/sysconfig/vha-agent
mycat /etc/default/vha-agent
mycat /etc/init.d/vha-agent
mycat /etc/vha-agent/nodes.conf
mycat /etc/varnish/nodes.conf
mycat /var/lib/vha-agent/vha-status

mycat /etc/sysconfig/varnish-agent
mycat /etc/default/varnish-agent
mycat /etc/init.d/varnish-agent
mycat /var/lib/varnish-agent/agent.param
mycat /var/lib/varnish-agent/boot.vcl
mycat /etc/varnish/varnish-agent.params

mycat /etc/hitch/hitch.conf

mycat /etc/varnish/modsec/modsecurity.conf

mycat /etc/init.d/vac
mycat /opt/vac/etc/defaults
mycat /var/opt/vac/log/vac-stderr.log
mycat /var/opt/vac/log/vac.log

mycat /var/log/mongodb/mongodb.log

# old vcs names
mycat /etc/sysconfig/vstatdprobe
mycat /etc/default/vstatdprobe
mycat /etc/init.d/vstatdprobe
mycat /etc/sysconfig/vstatd
mycat /etc/default/vstatd
mycat /etc/init.d/vstatd
mycat /etc/varnish/vstatd.params
mycat /etc/varnish/vstatdprobe.params

mycat /etc/hosts
mycat /etc/resolv.conf
mycat /etc/nsswitch.conf

run find /usr/local -name varnish
run find /var/lib/varnish -ls
run find /var/lib/varnish-agent -ls
run find /var/lib/varnish-controller -ls

# lsof
for pid in ${PID_ALL_VARNISHD}; do
	run lsof -p $pid
done

for a in /sys/block/*; do
	call_blockdev "$a"
done

run lsblk

#for d in $(lsblk -l -o NAME -n); do
#	run blockdev --getbsz /dev/$d
#done


for a in /var/log/messages /var/log/syslog; do
	if [ -r "$a" ]; then
		run egrep -i "(broadcaster|varnish|vha-agent|hitch|vac|rc.local|varnish-controller|vcs)" "$a"
	else
		incr
	fi
done

# ip tables
if  (lsmod | grep ip_tables > /dev/null); then
	run iptables-save
fi

if  (lsmod | grep ip6_tables > /dev/null); then
	run ip6tables-save
fi

# VCLs
for a in $(vcl_find); do
	mycat $a
done

# Modsec
for a in $(findmodsec); do
	mycat $a
done

# maps
for pid in ${PID_ALL_VARNISHD}; do
	runpipe "awk '$2 ~ \"rw\"' /proc/$pid/maps" "wc -l"
done

# Pick up the basic memory offset the text segments are in.
for pid in ${PID_ALL_VARNISHD}; do
	run grep "r-xp" /proc/$pid/maps
done

# Get cache memory maps
for pid in ${PID_CACHEMAIN}; do
	mycat /proc/$pid/smaps
	mycat /proc/$pid/status
done

if [ -z "${VARNISHADMARG}" ]; then
	banner "NO ADMINPORT SUPPLIED OR FOUND"
fi

# vadmin() tests for VARNISHADMARG as necessary

vadmin vcl.list

vcls="$(vadmin_getvcls)"
if [ -n "$vcls" ]; then
	case "$VARNISH" in
	*-4.0.[01]|*-[23].*)
		args=;;
	*)
		args="-v";;
	esac
	for vcl in $vcls; do
		vadmin vcl.show ${args} "$vcl"
	done
fi

vadmin param.show
vadmin param.show changed
case "$VARNISH" in
*-2.*)
	vadmin purge.list;;
*)
	vadmin ban.list;;
esac
case "$VARNISH" in
*-[23].*|*-4.0.*)
	vadmin debug.health;;
*)
	vadmin backend.list -p "*.*";;
esac
vadmin panic.show

case "$VARNISH" in
*-[23].*)
	args=;;
*)
	args="-g raw";;
esac
run timeout -s TERM 90 varnishlog -d ${args} -w "${DIR}/varnishlog.raw" $STATCMD

# perf
if [ "$PERF" = "TRUE" ]
then
	run timeout -s TERM 5 perf record -F 99 -p ${PID_ALL_VARNISHD_COMMA} -g
fi

if [ -d "/etc/api-engine" ]; then
	# Packages
	runpipe "rpm -qa" "grep api-engine"
	run egrep api-engine "/var/log/yum.log"

	# Permissions
	run find /var/lib/api-engine -ls
	run find /etc/api-engine -ls
	run find /var/log/api-engine -ls

	# Log files
	mycat /var/log/api-engine/api-engine-rest.log
	mycat /var/log/api-engine/api-engine-rest-uwsgi.log
	mycat /var/log/api-engine/api-engine-sync.log
	mycat /var/log/api-engine/api-engine-syncreload.log

	# Configuration files
	mycat /etc/api-engine/api-engine-rest.cfg
	mycat /etc/api-engine/api-engine-rest-uwsgi.ini
	mycat /etc/api-engine/api-engine-syncutil.cfg
	run egrep -v ^password "/etc/api-engine/api-engine-sync.cfg"

	# Generated VCL
	mycat /var/lib/api-engine/sync/vcl/current.vcl
	find /var/lib/api-engine/sync/vcl/*.vcl -type f | while read f; do
		mycat "$f"
	done
fi

# rc.local
if [ -f /etc/rc.local ]; then
	run cat /etc/rc.local
fi

run lspci -v -nn -k

cd "$ORIGPWD"
TGZ="varnishgather.${ID}.tar.gz"
tar czf "$TGZ" -C "$TOPDIR" "$RELDIR"

if [ -n "$UPLOAD" ]; then
	upload
	exit 0
fi

echo "==============================================================================="
echo "Please submit the file: $TGZ"
echo "==============================================================================="