testsuite

#!/bin/sh
#
# Anita test suite.  This mainly tests that installation of historic
# NetBSD versions still works.
#

python=python2.7

# Start background jobs this many seconds apart
spacing=30

while [ $# -gt 0 ]
do
    case $1 in
        --python)
            shift
            python="$1"
            ;;
        *)
            echo "unknown option $1" >&2
            exit 1
            ;;
    esac
    shift
done

set -e

logdir=log/testsuite.log.`date +%Y%m%d-%H%M%S`
echo "logging to $logdir"

mkdir -p $logdir

runtest() {
    echo "$@"
    mode=$1
    shift
    workdir=`$python ./anita print-workdir "$@"`
    logfile=$logdir/$workdir
    test ! -f $logfile || echo "warning: $logfile exists"
    rm -f $workdir/wd0.img
    (
        set -x -e
        $python ./anita --structured-log $mode --image-format sparse "$@"
        rm -f $workdir/wd0.img
        echo "passed"
    ) >$logfile 2>&1 &
    sleep $spacing
}

# Test local builds, if available
# disabled: 2020.10.07.19.06.50/alpha

for b in \
    2017.06.20.13.51.54/evbarm-earmv7hf \
    2020.09.25.19.24.56/evbarm-aarch64
do
    local_build=$(pwd)/test-data/local-builds/$b/
    if test -d $local_build
    then
        case $b in
            *evbarm-earmv7hf*) dtb="--dtb /usr/pkg/share/dtb/arm/vexpress-v2p-ca15-tc1.dtb" ;;
            *) dtb=""
        esac
        runtest boot $local_build $dtb
    else
        echo "no $local_build"
    fi
done

# Ditto for ISOs
# The sparc ISO is to test supplying entropy at install time

for b in \
    2020.10.07.19.06.50/images/NetBSD-9.99.73-hppa.iso \
    2020.11.11.07.34.55/release/images/NetBSD-9.99.75-sparc.iso
do
    local_iso=$(pwd)/test-data/local-builds/$b
    if test -f $local_iso
    then
        runtest boot $local_iso
    else
        echo "no $local_iso"
    fi
done

# Test an installation with debug symbols and partial source
# With 32M of memory, we hit the "stalled" problem (PR 47030)
debug_build=$(pwd)/test-data/local-builds/2017.06.20.13.51.54/i386/
if test -d $debug_build
then
    runtest "--memory-size=128M --disk-size=2G --sets=kern-GENERIC,etc,base,tests,games,misc,syssrc boot" $debug_build
fi

archive="https://archive.netbsd.org/pub/NetBSD-archive/"

runtest install $archive/NetBSD-2.1/i386/
runtest install $archive/NetBSD-3.0.1/i386/
runtest boot $archive/NetBSD-4.0/i386/
runtest boot $archive/NetBSD-5.0.2/i386/
runtest boot $archive/NetBSD-5.1.5/i386/

# These fail with "piixide0:0:0: lost interrupt" (tested with qemu 2.2.0)
# runtest boot http://ftp.fi.netbsd.org/pub/NetBSD/NetBSD-5.2/i386/
# runtest boot http://ftp.fi.netbsd.org/pub/NetBSD/NetBSD-5.2.3/i386/

runtest boot $archive/NetBSD-6.0/amd64/

# Test CD boot
runtest "--boot-from cdrom boot" $archive/NetBSD-7.2/amd64/

runtest boot $archive/NetBSD-8.2/amd64/

# Test newest version
runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-10.0/amd64/

# Test recent versions
runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-9.3/amd64/

# This panics with qemu 2.0.0, and fails due to the message
# "esp0: !TC on DATA XFER [intr 18, stat 82, step 4] prevphase 2, resid 0"
# being printed during set extraction with qemu 2.9.0.
#runtest boot http://ftp.fi.netbsd.org/pub/NetBSD/NetBSD-6.1.3/images/NetBSD-6.1.3-sparc.iso

# disabled while qemu sparc emulation is broken
#runtest boot http://ftp.netbsd.org/pub/NetBSD/iso/7.2/NetBSD-7.2-sparc.iso

runtest boot $archive/NetBSD-8.0/macppc/

runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-9.0/pmax/

# Installing hpcmips worked with GXemul 0.6.2 but it fails with 0.7.0.
#runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-9.0/hpcmips/

if /usr/bin/which simh-vax >/dev/null
then
    runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-9.0/images/NetBSD-9.0-vax.iso
else
    echo "no simh-vax"
fi

# 9.0 fails with assertion "(val & LASI_IRQ_BITS) == val", needs -current?
# runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-9.0/images/NetBSD-9.0-hppa.iso

# 9.0 hangs at "CTB: PRINTERPORT", need -current
#runtest boot http://ftp.netbsd.org/pub/NetBSD/NetBSD-9.0/alpha/

# sparc64 is broken with PR 54310, PR 54810

# evbarm-aarch64 lacks release images (there are dailies)

# Test selecting and installing the games set
url=$archive/iso/5.0.2/sparccd-5.0.2.iso
workdir=`$python ./anita print-workdir $url`
logfile=$logdir/$workdir
(
  set -x -e
  rm -f $workdir/wd0.img
  $python ./anita --sets=kern-GENERIC,base,etc,games --run "/usr/games/factor 1234567" boot $url | tee stdout.games
  grep "127 9721" stdout.games
  rm -f stdout.games
  rm -f $workdir/wd0.img
  echo "passed"
) >$logfile 2>&1 &

# Test running the ATF tests
url=http://ftp.netbsd.org/pub/NetBSD/NetBSD-archive/NetBSD-5.1/i386/
workdir=`$python ./anita print-workdir $url`
logfile=$logdir/$workdir
(
  set -x -e
  rm -f $workdir/wd0.img
  $python ./anita --sets=kern-GENERIC,etc,base,tests,games,misc test $url | tee stdout.atf || true
  # This is not to check that tests pass, but only that they at least run to completion
  grep 'passed test cases' stdout.atf
  rm -f stdout.atf
  rm -f $workdir/wd0.img
  echo "passed"
) >$logfile 2>&1 &

# Test anita 1.2 compatibility (in foreground)
logfile=$logdir/7.0.1
(
workdir=netbsd-7.0.1
rm -f $workdir/wd0.img
$python <<END
import anita
ver = anita.Release("7.0.1")
ver.install()
END
rm -f $workdir/wd0.img
echo "passed"
) >$logfile 2>&1 &

echo "waiting for tests to finish"
wait

pass=true
for f in $logdir/*
do
    if tail -1 $f | grep 'passed' >/dev/null
    then
        :
    else
        pass=false
        echo "failed: $f"
   fi
done

if $pass
then
echo "passed"
fi

$pass