chore: poc for visual regression testing

.circleci/config.yml

@@ -3,20 +3,20 @@ version: 2.1
 executors:
   node-browsers-small:
     docker:
-      - image: cimg/node:20.11-browsers
-    resource_class: small
+      - image: cimg/node:20.11-browsers@sha256:7ef58dedaa4596eae20162b790845aa0bdb54b5488fc7726dacfea39de3da2cd
+    resource_class: arm.small
     environment:
       NODE_OPTIONS: --max_old_space_size=2048
   node-browsers-medium:
     docker:
-      - image: cimg/node:20.11-browsers
-    resource_class: medium
+      - image: cimg/node:20.11-browsers@sha256:7ef58dedaa4596eae20162b790845aa0bdb54b5488fc7726dacfea39de3da2cd
+    resource_class: arm.medium
     environment:
       NODE_OPTIONS: --max_old_space_size=3072
   node-browsers-medium-plus:
     docker:
-      - image: cimg/node:20.11-browsers
-    resource_class: medium+
+      - image: cimg/node:20.11-browsers@sha256:7ef58dedaa4596eae20162b790845aa0bdb54b5488fc7726dacfea39de3da2cd
+    resource_class: arm.medium+
     environment:
       NODE_OPTIONS: --max_old_space_size=4096
   shellcheck:

.vscode/cspell.json

@@ -67,6 +67,8 @@
     "pageload",
     "petnames",
     "pipefail",
+    "pixelmatch",
+    "pngjs",
     "quickstart",
     "recompiles",
     "shellcheck",

development/lib/run-command.js

@@ -56,6 +56,7 @@ async function runCommand(command, args) {
      * was thrown inside the Promise constructor, the stack trace would show a few frames of
      * Node.js internals then end, without indicating where `runCommand` was called.
      */
+    console.log(error)
     if (error === internalError) {
       let errorMessage;
       if (errorCode !== null && errorSignal !== null) {
@@ -115,6 +116,7 @@ async function runInShell(command, args, output) {
       });
     });
   } catch (error) {
+    console.log(error)
     /**
      * The error is re-thrown here in an `async` context to preserve the stack trace. If this was
      * was thrown inside the Promise constructor, the stack trace would show a few frames of

package.json

@@ -55,6 +55,7 @@
     "test:e2e:firefox": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js",
     "test:e2e:firefox:flask": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js --build-type flask",
     "test:e2e:single": "node test/e2e/run-e2e-test.js",
+    "test:e2e:single:visual": "sh test/e2e/run-visual-test.sh",
     "test:coverage:mocha": "node ./test/run-unit-tests.js --mocha --coverage",
     "test:coverage:jest": "node ./test/run-unit-tests.js --jestGlobal --coverage",
     "test:coverage:jest:dev": "node ./test/run-unit-tests.js --jestDev --coverage",
@@ -473,6 +474,7 @@
     "@types/mocha": "^10.0.3",
     "@types/node": "^20",
     "@types/pify": "^5.0.1",
+    "@types/pixelmatch": "^5",
     "@types/pump": "^1.1.1",
     "@types/react": "^16.9.53",
     "@types/react-beautiful-dnd": "^13",
@@ -565,6 +567,8 @@
     "nock": "^13.2.9",
     "node-fetch": "^2.6.1",
     "nyc": "^15.1.0",
+    "pixelmatch": "^5.3.0",
+    "pngjs": "^7.0.0",
     "polyfill-crypto.getrandomvalues": "^1.0.0",
     "postcss": "^8.4.32",
     "postcss-rtlcss": "^4.0.9",

test/e2e/Dockerfile

@@ -0,0 +1,36 @@
+FROM cimg/node:20.11.0-browsers AS build
+
+WORKDIR '/usr/src/app'
+
+USER root
+COPY . .
+# Download and install Chrome
+RUN wget -q https://googlechromelabs.github.io/chrome-for-testing/#stable -O chrome.html \
+    && CHROME_VERSION=$(grep -o 'Stable<\/a><td><code>[0-9.]\+' chrome.html | sed 's/Stable<\/a><td><code>//') \
+    && CHROME_URL="https://storage.googleapis.com/chrome-for-testing-public/$CHROME_VERSION/linux64/chrome-linux64.zip" \
+    && wget -q "$CHROME_URL" -O chrome.zip \
+    && unzip -q chrome.zip \
+    && rm -f chrome.zip \
+    && mv chrome-linux64 /usr/local/bin/chrome &&\
+    chmod +x /usr/local/bin/chrome
+
+# Download the latest ChromeDriver
+RUN LATEST_CHROME_DRIVER_VERSION=$(curl -s https://googlechromelabs.github.io/chrome-for-testing/#stable | grep -o 'Stable<\/a><td><code>[0-9.]\+' | sed 's/Stable<\/a><td><code>//') && \
+    echo $LATEST_CHROME_DRIVER_VERSION &&\
+    wget -O /tmp/chromedriver.zip https://storage.googleapis.com/chrome-for-testing-public/$LATEST_CHROME_DRIVER_VERSION/linux64/chromedriver-linux64.zip && \
+    unzip /tmp/chromedriver.zip && \
+    mkdir -p /usr/local/bin && \
+    mv chromedriver-linux64/chromedriver /usr/local/bin/chromedriver &&\
+    chmod +x /usr/local/bin/chromedriver
+
+# Set environment variable for ChromeDriver
+ENV PATH="/usr/local/bin:${PATH}"
+
+# Install testing dependencies
+ENV HEADLESS=false
+ENV ENABLE_CHROME_LOGGING=true
+# ENV HEADLESS=true
+ENV NODE_OPTIONS="--dns-result-order=ipv4first"
+# ENV NVIDIA_VISIBLE_DEVICES=all
+
+ENTRYPOINT ["xvfb-run", "-a", "yarn", "test:e2e:single", "/usr/src/app/test/e2e/tests/account-menu/account-details.spec.js", "--browser=chrome"]

test/e2e/entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+xvfb-run
+
+# Run command 1
+yarn install
+
+# Run command 2
+yarn test:e2e:single test/e2e/tests/signature/signature-request.spec.js --browser=chrome

test/e2e/ganache.ts

@@ -19,6 +19,7 @@ export class Ganache {
 
     this.#server = server(options);
     await this.#server.listen(options.port);
+    console.log('Ganache server started');
   }
 
   getProvider() {

test/e2e/run-visual-test.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Stop the script if any command fails
+set -e
+
+# Define variables for better readability
+IMAGE_NAME=e2e/chrome:latest
+# CONTAINER_VOLUME_1=$(pwd)/test-artifacts/chrome/:/usr/src/app/test-artifacts/chrome
+CONTAINER_VOLUME_1=$(pwd)/:/usr/src/app/
+
+# copy build to the docker context
+# mkdir -p test/src
+#cp -r ./dist-test ./dist
+#cp -r ./builds-test ./builds
+# cp -r ./dist-test ./dist
+# cp -r ./builds-test ./builds
+# cp -r ./dist-test ../dist-test
+# cp -r ./builds-test ../builds-test
+# cp -r . .
+
+
+
+# Build the Docker image
+echo "Building the Docker image..."
+docker build -t $IMAGE_NAME test/e2e/
+
+# Check the script parameter
+UPDATE_SNAPSHOTS=""
+if [ "$1" == "update" ]; then
+    UPDATE_SNAPSHOTS="--update-snapshots"
+    echo " >> Updating snapshots!! Check screenshots change before you push them"
+fi
+
+# Run the Docker container
+echo "Running the Docker container..."
+result=$(docker run --init --platform linux/amd64 --rm -it --privileged -v "$CONTAINER_VOLUME_1" --network host $IMAGE_NAME $UPDATE_SNAPSHOTS | tee /dev/fd/2)
+if [ "$result" != "ok" ]; then
+    echo "Visual tests failed"
+fi
+
+# Remove the Docker image
+# echo "Removing the Docker image..."
+# docker image rm $IMAGE_NAME
+
+# # Remove files copied the building the image
+# echo "Removing dist/chrome from test dir..."
+# rm -rf /dist-test
+
+# echo "Script completed successfully."

test/e2e/tests/signature/signature-request.spec.js

@@ -1,4 +1,5 @@
 const { strict: assert } = require('assert');
+const path = require('path');
 const {
   withFixtures,
   regularDelayMs,
@@ -70,6 +71,7 @@ describe('Sign Typed Data Signature Request', function () {
         async ({ driver, ganacheServer }) => {
           const addresses = await ganacheServer.getAccounts();
           const publicAddress = addresses[0];
+          const baselinePath = path.resolve(__dirname, 'baselines');
           await unlockWallet(driver);
 
           await openDapp(driver);
@@ -84,6 +86,14 @@ describe('Sign Typed Data Signature Request', function () {
             windowHandles,
           );
 
+          // await driver.takeBaselineScreenshot(baselinePath, data.type); To take baselines
+
+          // Verify against baseline
+          await driver.matchBaselineScreenshot(
+            `${baselinePath}/${data.type}-screenshot`,
+            `${this.test.fullTitle()}`,
+          );
+
           await verifyAndAssertSignTypedData(
             driver,
             data.type,

test/e2e/webdriver/chrome.js

@@ -17,10 +17,17 @@ const HTTPS_PROXY_HOST = `${
 class ChromeDriver {
   static async build({ openDevToolsForTabs, port }) {
     const args = [
+      '--verbose',
+      '--no-sandbox',
+      '--ignore-certificate-errors',
+      '--disable-gpu',
       `--proxy-server=${HTTPS_PROXY_HOST}`, // Set proxy in the way that doesn't interfere with Selenium Manager
       '--disable-features=OptimizationGuideModelDownloading,OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints,NetworkTimeServiceQuerying', // Stop chrome from calling home so much (auto-downloads of AI models; time sync)
       '--disable-component-update', // Stop chrome from calling home so much (auto-update)
       '--disable-dev-shm-usage',
+      '--disable-gpu',
+      '--no-sandbox',
+      '--disable-software-rasterizer',
     ];
 
     if (process.env.MULTIPROVIDER) {

test/e2e/webdriver/driver.js

@@ -1,5 +1,7 @@
 const { promises: fs } = require('fs');
 const { strict: assert } = require('assert');
+const pixelmatch = require('pixelmatch');
+const { PNG } = require('pngjs');
 const {
   By,
   Condition,
@@ -658,6 +660,51 @@ class Driver {
     }
   }
 
+  async takeBaselineScreenshot(filePath, fileName) {
+    const filepathBase = `${filePath}/${fileName}`;
+    await fs.mkdir(filePath, { recursive: true });
+
+    const encodedString = await this.driver.takeScreenshot();
+    await fs.writeFile(`${filepathBase}-screenshot.png`, encodedString, {
+      encoding: 'base64',
+    });
+  }
+
+  async matchBaselineScreenshot(baselineImagePath, title) {
+    const artifactDir = `./test-artifacts/${this.browser}/${title}`;
+    const filepathBase = `${artifactDir}/currentScreen`;
+    await fs.mkdir(artifactDir, { recursive: true });
+
+    const encodedString = await this.driver.takeScreenshot();
+    await fs.writeFile(`${filepathBase}-screenshot.png`, encodedString, {
+      encoding: 'base64',
+    });
+
+    const baselineImage = PNG.sync.read(
+      await fs.readFile(`${baselineImagePath}.png`),
+    );
+    const currentImage = PNG.sync.read(
+      await fs.readFile(`${filepathBase}-screenshot.png`),
+    );
+    const { width, height } = baselineImage;
+    const difference = new PNG({ width, height });
+
+    const pixelDifference = pixelmatch(
+      baselineImage.data,
+      currentImage.data,
+      difference.data,
+      width,
+      height,
+      {
+        threshold: 0.1,
+      },
+    );
+
+    const differencePercent = (pixelDifference / (width * height)) * 100;
+    assert(differencePercent < 20);
+    await fs.writeFile(`${filepathBase}-diff.png`, PNG.sync.write(difference));
+  }
+
   // Error handling
 
   async verboseReportOnFailure(title, error) {

yarn.lock

@@ -10021,6 +10021,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/pixelmatch@npm:^5":
+  version: 5.2.6
+  resolution: "@types/pixelmatch@npm:5.2.6"
+  dependencies:
+    "@types/node": "npm:*"
+  checksum: 8299207286913414bfd95ac980418342f58e7bc93dc78befbd48ccab110b0341b400712f89fea2b0f0a3a4ad0b1aed18e98e26100b529abe60d4bcc14e11d2e3
+  languageName: node
+  linkType: hard
+
 "@types/prettier@npm:^2.7.2":
   version: 2.7.2
   resolution: "@types/prettier@npm:2.7.2"
@@ -24867,6 +24876,7 @@ __metadata:
     "@types/mocha": "npm:^10.0.3"
     "@types/node": "npm:^20"
     "@types/pify": "npm:^5.0.1"
+    "@types/pixelmatch": "npm:^5"
     "@types/pump": "npm:^1.1.1"
     "@types/react": "npm:^16.9.53"
     "@types/react-beautiful-dnd": "npm:^13"
@@ -24997,6 +25007,8 @@ __metadata:
     nyc: "npm:^15.1.0"
     obj-multiplex: "npm:^1.0.0"
     pify: "npm:^5.0.0"
+    pixelmatch: "npm:^5.3.0"
+    pngjs: "npm:^7.0.0"
     polyfill-crypto.getrandomvalues: "npm:^1.0.0"
     postcss: "npm:^8.4.32"
     postcss-rtlcss: "npm:^4.0.9"
@@ -27833,6 +27845,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pixelmatch@npm:^5.3.0":
+  version: 5.3.0
+  resolution: "pixelmatch@npm:5.3.0"
+  dependencies:
+    pngjs: "npm:^6.0.0"
+  bin:
+    pixelmatch: bin/pixelmatch
+  checksum: 10778aaa432211253ab0ae9160233d8aa56769ab6312b6bf8375100b67aaa126821626a0c3b433fb2a977864a8d2d145d754d4afa9ac14b84fcb1a0bdf98a4ae
+  languageName: node
+  linkType: hard
+
 "pkg-dir@npm:^3.0.0":
   version: 3.0.0
   resolution: "pkg-dir@npm:3.0.0"
@@ -27924,6 +27947,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pngjs@npm:^6.0.0":
+  version: 6.0.0
+  resolution: "pngjs@npm:6.0.0"
+  checksum: 692751ccd5e762623103900922caac982caa90258d9c6c04a6e2bc3397b1dedbaf9db826fc0fa068a29d607cad3df1d1eded0dec2ee35a0015c65cb5ef33ad18
+  languageName: node
+  linkType: hard
+
+"pngjs@npm:^7.0.0":
+  version: 7.0.0
+  resolution: "pngjs@npm:7.0.0"
+  checksum: e843ebbb0df092ee0f3a3e7dbd91ff87a239a4e4c4198fff202916bfb33b67622f4b83b3c29f3ccae94fcb97180c289df06068624554f61686fe6b9a4811f7db
+  languageName: node
+  linkType: hard
+
 "polished@npm:^4.2.2":
   version: 4.2.2
   resolution: "polished@npm:4.2.2"