Skip to content

Commit

Permalink
Merge branch 'release/1.2.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
seriema committed Jul 5, 2020
2 parents 82a902b + 85dd346 commit 900dcfc
Show file tree
Hide file tree
Showing 62 changed files with 1,345 additions and 394 deletions.
21 changes: 21 additions & 0 deletions .circleci/build-all-commits.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash

# Abort on error, error if variable is unset, and error if any pipeline element fails
set -euo pipefail

. ./helpers.sh

logfile=$(createLog circleci build)
branch=$(git rev-parse --abbrev-ref HEAD)
# Skip the latest commit as it will be automatically built by the branch being pushed.
mapfile -t commits < <(git log "develop...HEAD~1" --format="%h")

for commit in "${commits[@]}"
do
curl \
-u "${CIRCLECI_API_USER_TOKEN}:" \
--data "revision=$commit" \
--data "branch=$branch" \
"https://circleci.com/api/v1.1/project/github/seriema/retro-cloud/build" \
| tee -a "$logfile"
done
92 changes: 25 additions & 67 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,20 @@
version: 2
jobs:
bashValidation:
machine:
image: ubuntu-1604:201903-01
steps:

- checkout

- run:
name: Validate scripts execution permission
command: ./shared/validate-execute-permissions.sh

- run:
name: Lint Scripts
command: ./shared/lint-shellcheck.sh

scriptValidation:
docker:
- image: seriema/retro-cloud:develop
Expand All @@ -10,8 +25,8 @@ jobs:
- run:
name: Install Retro-Cloud on RaspberryPi (Setup Azure resources)
command: |
curl -OL "https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/raspberry-pi/download-and-run.sh"
bash download-and-run.sh $CIRCLE_SHA1
curl -fOL "https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/raspberry-pi/download-and-run.sh"
bash download-and-run.sh "$CIRCLE_SHA1" "CircleCI-$CIRCLE_BUILD_NUM"
rm download-and-run.sh
- run:
Expand All @@ -20,86 +35,28 @@ jobs:

- run:
name: Copy a freeware ROM to VM
command: bash -i ssh-vm.sh "curl -L https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-copy-rom.sh | bash -i"
command: bash -i ssh-vm.sh 'bash -i retro-cloud-setup/dev/test-copy-rom.sh'

- run:
name: Add Screenscraper.fr credentials to VM
command: |
bash -i ssh-vm.sh "echo '' | tee -a ~/.skyscraper/config.ini"
bash -i ssh-vm.sh "echo '[screenscraper]' | tee -a ~/.skyscraper/config.ini"
bash -i ssh-vm.sh "echo 'userCreds=\"${SCREENSCRAPER_USER}:${SCREENSCRAPER_KEY}\"' | tee -a ~/.skyscraper/config.ini"
command: ./add-scraper-credential.sh screenscraper "$SCREENSCRAPER_USER" "$SCREENSCRAPER_KEY"

- run:
name: Run scraper on VM
command: bash -i run-scraper.sh

- run:
name: Verify that ROM shows up in Azure File Share
command: |
curl -OL "https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/raspberry-pi/dev/test-az-share.ps1"
echo 'pwsh -executionpolicy bypass -File ".\test-az-share.ps1"' > run-az-share-test.sh
bash -i run-az-share-test.sh
rm run-az-share-test.sh test-az-share.ps1
- run:
name: Verify scraper output on VM
command: |
bash -i ssh-vm.sh "mkdir -p ~/tmp"
bash -i ssh-vm.sh "curl -L -o ~/tmp/test-gamelist.sh https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-gamelist.sh"
bash -i ssh-vm.sh "curl -L -o ~/tmp/test-gamelist.xml https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-gamelist.xml"
bash -i ssh-vm.sh "curl -L -o ~/tmp/test-gamelist-screenscraper-failed.xml https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/virtual-machine/dev/test-gamelist-screenscraper-failed.xml"
bash -i ssh-vm.sh "bash -i ~/tmp/test-gamelist.sh"
bash -i ssh-vm.sh "rm ~/tmp/test-gamelist.sh ~/tmp/test-gamelist.xml ~/tmp/test-gamelist-screenscraper-failed.xml"
- run:
name: Print RaspberryPi ~/.bashrc
command: cat ~/.bashrc
when: always

- run:
name: Print VM ~/.bashrc
command: |
echo 'ssh $RETROCLOUD_VM_USER@$RETROCLOUD_VM_IP "cat ~/.bashrc"' > print-vm-bashrc.sh
bash -i print-vm-bashrc.sh
rm print-vm-bashrc.sh
when: always

- run:
name: Print RaspberryPi ~/ directory listing
command: sudo find ~ | sed -e "s/[^-][^\/]*\// |/g" -e "s/|\([^ ]\)/|-\1/"
when: always

- run:
name: Print VM ~/ directory listing
command: |
printCmd='sudo find ~ | sed -e "s/[^-][^\/]*\// |/g" -e "s/|\([^ ]\)/|-\1/"'
echo 'ssh $RETROCLOUD_VM_USER@$RETROCLOUD_VM_IP' "'$printCmd'" > print-vm-filesystem.sh
bash -i print-vm-filesystem.sh
rm print-vm-filesystem.sh
when: always

- run:
name: Print Azure File Share directory listing
command: |
curl -OL "https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/raspberry-pi/dev/list-az-share.ps1"
echo 'pwsh -executionpolicy bypass -File ".\list-az-share.ps1"' > run-az-listing.sh
bash -i run-az-listing.sh
rm run-az-listing.sh list-az-share.ps1
when: always
name: Test all environments
command: bash -i retro-cloud-setup/dev/run-tests.sh

- run:
name: Print VM Skyscraper config
command: bash -i ssh-vm.sh "cat ~/.skyscraper/config.ini"
name: Print all environments
command: bash -i retro-cloud-setup/dev/print-all.sh
when: always

- run:
name: Teardown Azure resources
command: |
curl -OL "https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/raspberry-pi/teardown.sh"
curl -OL "https://raw.githubusercontent.com/seriema/retro-cloud/${CIRCLE_SHA1}/raspberry-pi/teardown-az.ps1"
bash -i teardown.sh
rm teardown.sh
rm teardown-az.ps1
command: cd retro-cloud-setup && bash teardown.sh "CircleCI-$CIRCLE_BUILD_NUM"
when: always

imageValidation:
Expand All @@ -118,5 +75,6 @@ workflows:
version: 2
commit:
jobs:
- bashValidation
- scriptValidation
- imageValidation
10 changes: 10 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,13 @@ indent_size = 4

[*.md]
trim_trailing_whitespace = false

# Always use 2 spaces for YAML files
# https://github.com/editorconfig/editorconfig/issues/329
[*.yml]
indent_style = space
indent_size = 2

# Editorconfig hadn't affected the PS scripts, so instead of reformatting...
[*.ps1]
indent_size = 2
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
logs/
.env
132 changes: 132 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
version: ~> 1.0 # Opt in to beta Build Config Validation https://docs.travis-ci.com/user/build-config-validation#beta-opt-in
language: shell # "Setting the language key to bash, sh or shell is equivalent to language: minimal." (https://docs.travis-ci.com/user/languages/minimal-and-generic/#aliases)
os: linux
dist: xenial

services:
- docker

env:
global:
- CONTAINER_NAME="Travis-$TRAVIS_BUILD_NUMBER"

jobs:
include:
- stage: "Workflows"
#############################################################################
# Workflow: bashValidation
# This must weirdly not have a dash, otherwise an empty job is created
script:
- ./lint.sh
name: "bashValidation"

#############################################################################
# Workflow: imageValidation
- script:
- ./docker/test.sh 'latest-amd64' 'seriema/retro-cloud'
name: "imageValidation"


#############################################################################
# Workflow: scriptValidation
- script:
# Start Docker
- FOLD_START docker_start "Start Docker"
- |
docker pull seriema/retro-cloud:latest-amd64 \
&& docker container create --cap-add SYS_ADMIN --device /dev/fuse --env-file .env --interactive --rm --tty --name "$CONTAINER_NAME" --workdir /home/pi seriema/retro-cloud:latest-amd64 \
&& docker container start "$CONTAINER_NAME" \
; FOLD_END
# We won't be stopping the container. Seems pointless as Travis will shut down the VM that's running the container anyway.

# Mimic a user's RetroPie by including some ROMs etc
- FOLD_START rpi_user "Mimic a user's RetroPie by including some ROMs etc"
- docker exec "$CONTAINER_NAME" /bin/bash -euo pipefail -c "curl -fL 'https://raw.githubusercontent.com/seriema/retro-cloud/${TRAVIS_COMMIT}/raspberry-pi/dev/download-freeware-roms.sh' | bash" ; FOLD_END

# Install Retro-Cloud on RaspberryPi (Setup Azure resources)
- FOLD_START rpi_setup "Install Retro-Cloud on RaspberryPi (Setup Azure resources)"
- |
docker exec "$CONTAINER_NAME" /bin/bash -euo pipefail -c " \
curl -fOL 'https://raw.githubusercontent.com/seriema/retro-cloud/${TRAVIS_COMMIT}/raspberry-pi/download-and-run.sh' \
&& bash download-and-run.sh $TRAVIS_COMMIT $CONTAINER_NAME \
&& rm download-and-run.sh" \
; FOLD_END
# Install Retro-Cloud on VM
- FOLD_START vm_setup "Install Retro-Cloud on VM"
- docker exec "$CONTAINER_NAME" /bin/bash -c "bash -i setup-vm.sh $TRAVIS_COMMIT" ; FOLD_END

# Copy user's ROMs to Retro-Cloud
- FOLD_START copy_rom "Copy user's ROMs to Retro-Cloud"
- docker exec "$CONTAINER_NAME" /bin/bash -c 'bash -i copy-roms-to-file-share.sh' ; FOLD_END

# Add user's Screenscraper.fr credentials to VM
- FOLD_START screenscraper "Add Screenscraper.fr credentials to VM"
- docker exec "$CONTAINER_NAME" /bin/bash -c "./add-scraper-credential.sh screenscraper '$SCREENSCRAPER_USER' '$SCREENSCRAPER_KEY'" ; FOLD_END

# Run scraper on VM
- FOLD_START run_scraper "Run scraper on VM"
- docker exec "$CONTAINER_NAME" /bin/bash -c 'bash -i run-scraper.sh' ; FOLD_END

# Run all tests
- FOLD_START test "Run all tests"
- docker exec "$CONTAINER_NAME" /bin/bash -c 'bash -i retro-cloud-setup/dev/run-tests.sh' ; FOLD_END

# Print all environments
- FOLD_START print "Print all environments"
- docker exec "$CONTAINER_NAME" /bin/bash -c 'bash -i retro-cloud-setup/dev/print-all.sh' ; FOLD_END

# Teardown Azure resources
- FOLD_START az_teardown "Teardown Azure resources"
- docker exec "$CONTAINER_NAME" /bin/bash -c "cd retro-cloud-setup && bash teardown.sh '$CONTAINER_NAME'" ; FOLD_END

before_script:
- |
# Create an .env file to use with Docker
echo "AZURE_TENANT_ID=$AZURE_TENANT_ID" > .env
echo "AZURE_SERVICE_PRINCIPAL_USER=$AZURE_SERVICE_PRINCIPAL_USER" >> .env
echo "AZURE_SERVICE_PRINCIPAL_SECRET=$AZURE_SERVICE_PRINCIPAL_SECRET" >> .env
echo "SCREENSCRAPER_USER=$SCREENSCRAPER_USER" >> .env
echo "SCREENSCRAPER_KEY=$SCREENSCRAPER_KEY" >> .env
- |
# Load convenience functions for Travis fold
# Based on these scripts:
# * https://github.com/spotify/ios-ci/blob/master/bin/travis_fold
# * https://www.koszek.com/blog/2016/07/25/dealing-with-large-jobs-on-travis/
tmpFoldName=/tmp/.travis_fold_name
tmpFoldHeading=/tmp/.travis_fold_heading
FOLD_START() {
local action=start
local name=$1
# Add a lot of space (tabs) to heading to extend the background color
local heading=" $2 \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t"
# Gray bg, Yellow fg, and use \r to overwrite the "travis_fold:action:name" as it's just annoying
echo -en "travis_fold:${action}:${name}\r\e[40;33m${heading}\n"
# Save the current scope, so we can call FOLD_END without passing them again
/bin/echo -n "$name" > "$tmpFoldName"
/bin/echo -n "$heading" > "$tmpFoldHeading"
# This puts the "FOLD_START" timer inside the fold which looks better than without this line (then the timer is outside)
travis_time_start
}
FOLD_END() {
local exitCode=$?
local action=end
local name=$(cat "${tmpFoldName}")
local heading=$(cat "${tmpFoldHeading}")
travis_time_finish
# Black bg, Grey fg, and use \r to overwrite the "travis_fold:action:name" as it's just annoying
echo -en "travis_fold:${action}:${name}\r\e[100;30m${heading}\n"
# Return the exit code otherwise the step won't fail if there was a failure
return "$exitCode"
}
name: "scriptValidation"
32 changes: 32 additions & 0 deletions .travisci/build-all-commits.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/bin/bash

# Abort on error, error if variable is unset, and error if any pipeline element fails
set -euo pipefail

. ./helpers.sh

logfile=$(createLog circleci build)
branch=$(git rev-parse --abbrev-ref HEAD)
# Skip the latest commit as it will be automatically built by the branch being pushed.
mapfile -t commits < <(git log "develop...HEAD~1" --format="%h")

# To run some commits manually:
# commits=(0f143b4 abc1234)

for commit in "${commits[@]}"
do
body="{
\"request\": {
\"branch\":\"$branch\",
\"sha\":\"$commit\"
}}"

curl -X POST \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-H "Travis-API-Version: 3" \
-H "Authorization: token $TRAVISCI_API_USER_TOKEN" \
-d "$body" \
'https://api.travis-ci.com/repo/seriema%2Fretro-cloud/requests' \
| tee -a "$logfile"
done
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ RUN if [ "$(uname -m)" = 'armv7l' ]; then \
# https://raspberrypi.stackexchange.com/questions/78427/what-repository-to-add-for-apt-to-find-raspberrypi-kernel
&& echo "deb http://raspbian.raspberrypi.org/raspbian/ stretch main contrib non-free rpi" >> /etc/apt/sources.list \
&& echo "deb http://archive.raspberrypi.org/debian/ stretch main ui" >> /etc/apt/sources.list.d/raspi.list \
&& curl -L http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \
&& curl -fL http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - \
# https://www.raspbian.org/RaspbianRepository
&& curl -L http://archive.raspbian.org/raspbian.public.key | apt-key add - \
&& curl -fL http://archive.raspbian.org/raspbian.public.key | apt-key add - \
#
# 3. Refresh the source list to make sure it worked
&& apt-get update; \
Expand Down

0 comments on commit 900dcfc

Please sign in to comment.