Merge branch 'release/1.1.0'

.circleci/config.yml

@@ -1,6 +1,6 @@
 version: 2
 jobs:
-  infra:
+  scriptValidation:
     docker:
     - image: seriema/retro-cloud:develop
     shell: /bin/bash --login -eo pipefail
@@ -44,6 +44,7 @@ jobs:
     - run:
         name: Verify scraper output on VM
         command: |
+            bash -i ssh-vm.sh "mkdir -p ~/tmp"
             bash -i ssh-vm.sh "curl -L -o ~/tmp/test-gamelist.sh https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-gamelist.sh"
             bash -i ssh-vm.sh "curl -L -o ~/tmp/test-gamelist.xml https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-gamelist.xml"
             bash -i ssh-vm.sh "curl -L -o ~/tmp/test-gamelist-screenscraper-failed.xml https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-gamelist-screenscraper-failed.xml"
@@ -101,8 +102,21 @@ jobs:
             rm teardown-az.ps1
         when: always
 
+  imageValidation:
+    docker:
+    - image: seriema/retro-cloud:develop
+    working_directory: /home/pi/retro-cloud
+    steps:
+
+    - checkout
+
+    - run:
+        name: Validate Docker image
+        command: bash docker/compose/run_tests.sh
+
 workflows:
   version: 2
-  build:
+  commit:
     jobs:
-    - infra
+    - scriptValidation
+    - imageValidation

.dockerignore

@@ -1,2 +1 @@
 *
-!docker/

Dockerfile

@@ -1,7 +1,10 @@
 #
 # CREATE A Fake Raspbian IMAGE
 #
-FROM ubuntu:18.04 AS raspberrypi
+FROM debian:stretch AS raspberrypi
+
+# Mentioned in https://hub.docker.com/_/debian and seen in https://github.com/laseryuan/docker-apps/blob/db3c154ebe/retropie/Dockerfile.templ#L13
+ENV LANG C.UTF-8
 
 # Install prerequisites
 RUN apt-get update \
@@ -10,14 +13,60 @@ RUN apt-get update \
     # Required to run image as non-root user
     sudo
 
-# skel files are used as the basis for new users and include .bashrc etc:
-# https://www.raspberrypi.org/documentation/linux/usage/users.md
-# These files were copied from a RetroPie 4.5.1 on a RaspberryPi 3 and
-# are included to make the image more realistic. One difference from
-# base docker image is that .bashrc has colors enabled.
-# Note: Windows `COPY` sets execute permission (-rwxr-xr-x), which needs to be reverted.
-COPY docker/rpi/etc/skel /etc/skel
-RUN if [ "$(uname -m)" = 'x86_64' ]; then chmod -R -x+X /etc/skel; fi
+# Get rid of the warning: "debconf: unable to initialize frontend: Dialog"
+# https://serverfault.com/a/797318/565229
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Get rid of the warning: "debconf: delaying package configuration, since apt-utils is not installed"
+RUN apt-get install -y apt-utils
+
+# Add the raspberrypi.org and raspbian.org sources for packages expected by RetroPie-Setup when running on a RaspberryPi.
+# Causes warnings on x86_64 (amd64) so it has to be conditional. Example: "N: Skipping acquire of configured file [...] as repository 'http://raspbian.raspberrypi.org/raspbian stretch InRelease' doesn't support architecture 'amd64'"
+RUN if [ "$(uname -m)" = 'armv7l' ]; then \
+        #
+        # 1. Get the packages needed to add sources
+        apt-get update && apt-get install -y \
+        # curl is used to download the public keys in the next step (and used by the retro-cloud scripts)
+        curl \
+        # gnupg is used by apt-key in the next step
+        gnupg \
+        #
+        # 2. Add the sources and their public keys
+        # https://raspberrypi.stackexchange.com/questions/78427/what-repository-to-add-for-apt-to-find-raspberrypi-kernel
+        && echo "deb http://raspbian.raspberrypi.org/raspbian/ stretch main contrib non-free rpi" >> /etc/apt/sources.list \
+        && echo "deb http://archive.raspberrypi.org/debian/ stretch main ui" >> /etc/apt/sources.list.d/raspi.list \
+        && curl -L http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \
+        # https://www.raspbian.org/RaspbianRepository
+        && curl -L http://archive.raspbian.org/raspbian.public.key | apt-key add - \
+        #
+        # 3. Refresh the source list to make sure it worked
+        && apt-get update; \
+    fi
+
+#
+# Create a fat cache layer with most packages found per platform.
+# Greatly reduces build time spent on the RetroPie-Setup layer (from >20min to <5min).
+# WARNING! Rebuilding this cache layer takes a very long time! Don't modify it often.
+RUN if [ "$(uname -m)" = 'armv7l' ]; then \
+        apt-get update && apt-get install -y \
+        autoconf automake autopoint autotools-dev binutils bison bsdmainutils build-essential bzip2 ca-certificates cmake cpp dbus debhelper device-tree-compiler devscripts dh-autoreconf dh-strip-nondeterminism dialog dirmngr distro-info-data dpkg-dev fbi fbset fcitx-bin fcitx-libs-dev file flex fontconfig fontconfig-config fonts-dejavu-core fonts-freefont-ttf g++ gcc gettext gettext-base ghostscript git gnupg gpg groff-base insserv intltool-debian krb5-locales less libasound2-dev libavcodec-dev libavdevice-dev libavformat-dev libcurl4-openssl-dev libdbus-1-dev libegl1-mesa-dev libfreeimage-dev libfreetype6-dev libgbm-dev libgl1-mesa-dev libgles2-mesa-dev libglu1-mesa-dev libibus-1.0-dev libjpeg-dev libraspberrypi-bin libraspberrypi-dev libraspberrypi-doc libsamplerate0-dev libsndio-dev libspeexdsp-dev libudev-dev libusb-1.0-0-dev libvlccore-dev libvlc-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev libxss-dev libxt-dev libxv-dev libxxf86vm-dev lsb-release m4 make mc mesa-common-dev meson mime-support netbase ninja-build omxplayer openssh-client openssl patch perl pinentry-curses pkg-config po-debconf poppler-data procps publicsuffix python python2.7 python3 python-pyudev qdbus qtchooser qtcore4-l10n rapidjson-dev raspberrypi-bootloader readline-common shared-mime-info systemd ucf udev unzip vlc wget x11-common xauth xkb-data xmlstarlet xorg-sgml-doctools xtrans-dev xz-utils zlib1g-dev \
+        ; \
+    fi
+RUN if [ "$(uname -m)" = 'x86_64' ]; then \
+        # Note: The N64 emulator mupen64plus is used on amd64 but requires a newer version of cmake (3.9+) than
+        # is available (3.7.2) on debian:stretch, so it will be installed separately.
+        apt-get update && apt-get install -y \
+        adduser adwaita-icon-theme apt aspell aspell-en at-spi2-core base-files base-passwd bash binutils bison bsdmainutils bsdutils build-essential bzip2 ca-certificates coreutils cpp cron dash dbus dbus-user-session dconf-gsettings-backend dconf-service debconf debianutils desktop-file-utils dialog dictionaries-common diffutils dirmngr distro-info-data dmsetup dosfstools dpkg dpkg-dev e2fsprogs eject emacsen-common enchant fakeroot feh file findutils flex fontconfig fontconfig-config fonts-dejavu-core fonts-freefont-ttf freeglut3 fuse g++ gcc gdisk git glib-networking glib-networking-common glib-networking-services gnome-terminal gnome-terminal-data gnupg gpg grep groff-base gsettings-desktop-schemas gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-pulseaudio gstreamer1.0-x gtk-update-icon-cache gvfs gvfs-common gvfs-daemons gvfs-libs gzip hicolor-icon-theme hostname hunspell-en-us i965-va-driver init-system-helpers iso-codes krb5-locales less libasound2-dev libavcodec-dev libavdevice-dev libavformat-dev libboost-filesystem-dev libcurl4-openssl-dev libfreeimage-dev libfreetype6-dev libglew-dev libpulse-dev libsamplerate0-dev libsdl2-dev libspeexdsp-dev libudev-dev libusb-1.0-0-dev libvlccore-dev libvlc-dev libvulkan-dev libxkbcommon-dev m4 make man-db manpages manpages-dev mawk mc mc-data mesa-common-dev mesa-va-drivers mesa-vdpau-drivers meson mime-support mount multiarch-support nasm ncurses-base ncurses-bin netbase ninja-build notification-daemon ntfs-3g openssh-client openssl parted passwd patch perl pinentry-curses pkg-config policykit-1 powermgmt-base procps publicsuffix python python2.7 python3 python-apt-common python-minimal python-pyudev python-six python-talloc qt5-gtk-platformtheme qttranslations5-l10n rapidjson-dev readline-common samba-libs sed sensible-utils shared-mime-info software-properties-common sshfs sudo systemd systemd-sysv sysvinit-utils tar ucf udev udisks2 unattended-upgrades unzip util-linux va-driver-all vdpau-driver-all vlc vlc-bin vlc-data vlc-l10n vlc-plugin-base vlc-plugin-notify vlc-plugin-qt vlc-plugin-samba vlc-plugin-skins2 vlc-plugin-video-output vlc-plugin-video-splitter vlc-plugin-visualization wget x11-common xauth xdg-user-dirs xdg-utils xkb-data xmlstarlet xorg-sgml-doctools xtrans-dev xz-utils yelp yelp-xsl yudit-common zlib1g zlib1g-dev \
+        ; \
+        # Install cmake 3.9+ (which depends on libarchive13): https://backports.debian.org/
+        echo "deb http://deb.debian.org/debian stretch-backports-sloppy main" | sudo tee -a /etc/apt/sources.list >/dev/null \
+        && sudo apt-get update \
+        && sudo apt-get -t stretch-backports-sloppy install -y libarchive13 \
+        && echo "deb http://deb.debian.org/debian stretch-backports main" | sudo tee -a /etc/apt/sources.list >/dev/null \
+        && sudo apt-get update \
+        && sudo apt-get -t stretch-backports install -y cmake \
+        ; \
+    fi
 
 # Mimic RaspberryPi: Create a user called "pi" without a password that's in the groups pi and sudo
 # Use `adduser` instead of `useradd`:
@@ -38,21 +87,28 @@ WORKDIR /home/pi
 
 ## Install RetroPie ##
 
-# Install the needed packages for the RetroPie setup script on Debian/Ubuntu:
-# https://retropie.org.uk/docs/Debian/
-RUN sudo apt-get update \
-    && sudo apt-get install -y \
-    git dialog unzip xmlstarlet
+# Install the needed packages for the RetroPie setup script on Raspbian and Debian/Ubuntu respectively.
+# RPi: https://retropie.org.uk/docs/Manual-Installation/
+RUN if [ "$(uname -m)" = 'armv7l' ]; then sudo apt-get update && sudo apt-get install -y git lsb-release; fi
+# Linux: https://retropie.org.uk/docs/Debian/
+RUN if [ "$(uname -m)" = 'x86_64' ]; then sudo apt-get update && sudo apt-get install -y git dialog unzip xmlstarlet; fi
 
 # Download the latest RetroPie setup script:
-RUN git clone --depth=1 https://github.com/RetroPie/RetroPie-Setup.git
+RUN git clone https://github.com/RetroPie/RetroPie-Setup.git
 
 # Enter the folder with the setup script
 WORKDIR /home/pi/RetroPie-Setup
 
+# Checkout a specific version to avoid sudden upgrades that break the image
+# 4.5.17 + error code fix https://github.com/RetroPie/RetroPie-Setup/commit/50e8300
+RUN git checkout 50e8300
+
 # Install RetroPie
-# WARNING! This takes hours. Changing anything above this point in the Dockerfile will invalidate the cache of this layer, forcing an install.
-RUN sudo ./retropie_packages.sh setup basic_install
+# WARNING! Rebuilding this cache layer takes a very long time! Don't modify it often.
+RUN if [ "$(uname -m)" = 'armv7l' ]; then sudo __platform="rpi3" ./retropie_packages.sh setup basic_install; fi
+RUN if [ "$(uname -m)" = 'x86_64' ]; then sudo ./retropie_packages.sh setup basic_install; fi
+# The lines above can be commented out to speed up builds for testing, but then needs the line below.
+# RUN sudo mkdir -p /opt/retropie/configs/all && sudo chmod g+w -R /opt/retropie/configs/all
 
 # Exit the folder with the setup script
 WORKDIR /home/pi

README.md

@@ -20,6 +20,8 @@ An expensive and over-engineered approach to storing ROMs and their metadata whi
     $ curl -sSL https://raw.githubusercontent.com/seriema/retro-cloud/master/raspberry-pi/download-and-run.sh | bash
     # Or this shortened URL:
     $ curl -sSL https://tiny.cc/retro-cloud-setup | bash
+    # Or get the latest development version:
+    $ curl -sSL https://tiny.cc/rc-rpi | bash
     ```
 
     > **NOTE!** You will be prompted to log into your Azure account. The script pauses with the message:
@@ -39,6 +41,8 @@ An expensive and over-engineered approach to storing ROMs and their metadata whi
         $ curl -sSL https://raw.githubusercontent.com/seriema/retro-cloud/master/virtual-machine/setup.sh | bash
         # Or this shortened URL:
         $ curl -sSL https://tiny.cc/retro-cloud-setup-vm | bash
+        # Or get the latest development version:
+        $ curl -sSL https://tiny.cc/rc-vm | bash
         ```
 
 1. Copy ROMs to Azure File Share. Alternatives:
@@ -63,6 +67,8 @@ An expensive and over-engineered approach to storing ROMs and their metadata whi
     * `git clone git@github.com:seriema/retro-cloud.git && cd retro-cloud && git checkout develop`
 * Testing
     * `docker run --privileged -it --rm seriema/retro-cloud:develop`
+    * `docker/compose/run_tests.sh`
+    * `docker-compose -f docker-compose.test.yml up`
 * Docker
     * `docker build -t seriema/retro-cloud:amd64 .`
     * `docker push seriema/retro-cloud:amd64`

docker-compose.test.yml

@@ -0,0 +1,6 @@
+sut:
+    build: .
+    working_dir: /home/pi/retro-cloud-test
+    volumes:
+        - ./docker/compose:/home/pi/retro-cloud-test/docker/compose
+    command: bash docker/compose/run_tests.sh

docker/compose/directory-listing.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Abort on error, error if variable is unset, and enable debug output
+set -eux
+
+echo 'Verify number of roms folders. There is one for each successful emultator installed.'
+# Added when RetroPie-Setup was failing silently and not installing all emulators.
+# Note: Export a new list with: $ ls -m ~/RetroPie/roms
+if [[ $(uname -m) = 'armv7l' ]]; then
+    emulators="amiga, amstradcpc, arcade, atari2600, atari5200, atari7800, atari800, atarilynx, fba, fds, gamegear, gb, gba, gbc, genesis, mame-libretro, mame-mame4all, mastersystem, megadrive, n64, neogeo, nes, ngp, ngpc, pcengine, psx, sega32x, segacd, sg-1000, snes, vectrex, zxspectrum";
+else
+    # Two emulators aren't available on amd64: mame-mame4all, amiga
+    emulators="amstradcpc, arcade, atari2600, atari5200, atari7800, atari800, atarilynx, fba, fds, gamegear, gb, gba, gbc, genesis, mame-libretro, mastersystem, megadrive, n64, neogeo, nes, ngp, ngpc, pcengine, psx, sega32x, segacd, sg-1000, snes, vectrex, zxspectrum";
+fi
+[[ -z $(echo $(echo $emulators | tr ',' '\n') $(ls -1 ~/RetroPie/roms) | tr ' ' '\n' | sort | uniq -u) ]]
+
+echo 'Verify that there are no builds left. There is one for each failed build.'
+# Added when RetroPie-Setup was failing silently and not installing all emulators.
+[[ -z $(find /home/pi/RetroPie-Setup/tmp/build -maxdepth 3 -type f) ]]
+
+echo 'Verify that line endings of all files are LF. Windows uses CRLF which can get copied over by Docker ADD/COPY.'
+# Added when running the container would throw '\r' parse errors when the users bash profile. Despite
+# .gitattribute set to checkout all files in docker/rpi as LF it could still check them out as CRLF,
+# and Docker has a tendency to use the host line endings when copying files.
+# Note: The Dockerfile no longer copies bash files because COPY constantly invalidates the cache, forcing
+# unnecessary rebuilds that take 30-60 minutes locally and 2-3 hours on Docker Hub. The test is kept
+# as a regression test for future changes to the Dockerfile.
+[[ -z $(grep -r $'\r' * -l) ]]

docker/compose/packages.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Abort on error, error if variable is unset, and enable debug output
+set -eux
+
+echo 'Verify that apt-get is not broken due to wrong package sources, repo public keys, etc.'
+# Added when apt-get would not update on Windows because there were Raspberry Pi Foundation apt-get
+# sources expecting arm32, and when not having the right public keys validating those sources.
+[[ $(sudo apt-get update) ]]

docker/compose/run_tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Abort on error, error if variable is unset, and enable debug output
+set -eux
+
+echo 'TEST: directory listing'
+./docker/compose/directory-listing.sh
+
+echo 'TEST: packages'
+./docker/compose/packages.sh
+
+echo 'TEST: user access'
+./docker/compose/user-access.sh
+
+echo 'TEST: Done.'

docker/compose/user-access.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Abort on error, error if variable is unset, and enable debug output
+set -eux
+
+echo 'Verify access permissions of different directories'
+# Added when the user was created as root so the user didn't have regular access to $HOME.
+[[ $(stat -c %a /home/pi) -eq 755 ]]
+# Added when RetroPie-Setup was running as another user than expected so the user didn't have access.
+[[ $(stat -c %a /home/pi/RetroPie) -eq 755 ]]
+# Added when 'git clone' required chmod but shouldn't have needed to.
+[[ $(stat -c %a /home/pi/RetroPie-Setup) -eq 755 ]]
+# Note: /dev/fuse test is only available at runtime so this test doesn't test the image. It would
+# test the running container. It's only a problem when running the container on Windows/amd64 but
+# it means that mounting won't work and running raspberry-pi/mount-vm-share.sh will fail.
+# [[ $(stat -c %a /dev/fuse) -eq 666 ]]
+
+echo 'Verify group memberships'
+# Added when replacing 'useradd' with 'adduser' to verify that the user gets a group with the same name.
+[[ $(groups | grep pi) ]]
+# Added when replacing 'useradd' with 'adduser' to verify that the user is still given sudo rights.
+[[ $(groups | grep sudo) ]]
+
+echo 'Verify that the image default user is "pi"'
+# Added when not setting "USER pi" as the last user in the Dockerfile.
+[[ $(whoami) = "pi" ]]