Update tools to latest versions (#756)

rellic: * Updated to latest `master` commit * Remove `|| true` to make sure tests stay passing remill: * Updated to latest `master` commit * Updated vcpkg portfile to latest released version 4.0.24 * Allow failure of tests on aarch64. See lifting-bits/remill#518 anvill: * Updated to latest `master` commit * Add other submodule for anvill version compatible with mcsema * Don't install Python3 components since we don't test them mcsema: * Updated to latest `master` commit * Add pinned anvill version submodule * Add two smoketests * Allow failure for `x86` smoketest on aarch64 host. See lifting-bits/mcsema#753
lifting-bits · May 27, 2021 · bda1e09 · bda1e09
1 parent d9417f2
commit bda1e09
Show file tree

Hide file tree

Showing 12 changed files with 129 additions and 25 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -12,3 +12,6 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    ignore:
+      # McSema has a pinned version of anvill
+      - dependency-name: "anvill-mcsema"
diff --git a/.github/workflows/vcpkg_ci_linux.yml b/.github/workflows/vcpkg_ci_linux.yml
@@ -78,7 +78,7 @@ jobs:
 
       - uses: actions/checkout@v2.3.4
         with:
-          submodules: true
+          submodules: recursive
 
       - name: Read vcpkg Info
         id: vcpkg_info
@@ -139,7 +139,7 @@ jobs:
             @dependencies.txt
 
       - name: 'Export Packages'
-        if: contains(github.event.head_commit.message, 'debug artifacts') || github.event.release
+        if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release
         shell: 'bash'
         run: |
           apt-get update
@@ -150,7 +150,7 @@ jobs:
           ./emit_artifacts.sh ${{ env.ARTIFACT_NAME }}
 
       - uses: actions/upload-artifact@v2.2.3
-        if: "contains(github.event.head_commit.message, 'debug artifacts')"
+        if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.ARTIFACT_NAME }}
@@ -210,7 +210,7 @@ jobs:
             ..
           cmake --build .
           cmake --build . --target install
-          ../scripts/roundtrip.py ./tools/rellic-decomp-* ../tests/tools/decomp "${VCPKG_ROOT}/installed/${{ matrix.host.triplet }}/tools/${{ matrix.llvm }}/clang" || true
+          ../scripts/roundtrip.py ./tools/rellic-decomp-* ../tests/tools/decomp "${VCPKG_ROOT}/installed/${{ matrix.host.triplet }}/tools/llvm/clang"
 
       - name: 'Test remill build'
         shell: 'bash'
@@ -226,7 +226,13 @@ jobs:
           cmake --build .
           cmake --build . --target install
           cmake --build . --target test_dependencies
-          env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test || true
+          # See issue https://github.com/lifting-bits/remill/issues/518 for
+          # failing aarch64 tests
+          if [ ${{ matrix.host.arch }} == "arm64" ] ; then
+            env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test || true
+          else
+            env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test
+          fi
 
       # Only for LLVM-11 right now...
       - name: 'Anvill build - vcpkg remill'
@@ -240,6 +246,7 @@ jobs:
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DVCPKG_ROOT="${VCPKG_ROOT}" \
             -DANVILL_ENABLE_INSTALL_TARGET=ON \
+            -DANVILL_ENABLE_PYTHON3_LIBS=OFF \
             -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
             ..
           cmake --build .
@@ -256,6 +263,26 @@ jobs:
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
+            -DVCPKG_ROOT="${VCPKG_ROOT}" \
+            -DANVILL_ENABLE_INSTALL_TARGET=ON \
+            -DANVILL_ENABLE_PYTHON3_LIBS=OFF \
+            -Dremill_DIR="$(pwd)/../../remill/build/install/lib/cmake/remill" \
+            ..
+          cmake --build .
+          cmake --build . --target install
+          ./install/bin/anvill-decompile-json-* -spec ../tools/decompile-json/tests/specs/ret0.json -bc_out ./ret0.bc -ir_out ret0.ir
+
+      - name: 'Anvill-McSema build - custom remill'
+        shell: 'bash'
+        if: ${{ matrix.llvm != 'llvm-12' }}
+        run: |
+          cd anvill-mcsema
+          rm -rf build
+          mkdir -p build && cd build
+          cmake -G Ninja \
+            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
+            -DVCPKG_ROOT="${VCPKG_ROOT}" \
             -DANVILL_ENABLE_INSTALL_TARGET=ON \
             -Dremill_DIR="$(pwd)/../../remill/build/install/lib/cmake/remill" \
             ..
@@ -265,20 +292,31 @@ jobs:
 
       - name: 'Test mcsema build'
         shell: 'bash'
-        if: ${{ matrix.llvm == 'llvm-11' }}
+        if: ${{ matrix.llvm != 'llvm-12' }}
         run: |
           cd mcsema
           rm -rf build
           mkdir -p build && cd build
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
+            -DVCPKG_ROOT="${VCPKG_ROOT}" \
             -Dremill_DIR="$(pwd)/../../remill/build/install/lib/cmake/remill" \
-            -Danvill_DIR="$(pwd)/../../anvill/build/install/lib/cmake/anvill" \
+            -Danvill_DIR="$(pwd)/../../anvill-mcsema/build/install/lib/cmake/anvill" \
             ..
           cmake --build .
           cmake --build . --target install
 
+          ./install/bin/mcsema-lift-* --version
+          ./install/bin/mcsema-lift-* --arch amd64 --os linux --cfg ../tests/test_suite_generator/generated/prebuilt_cfg/amd64/linux/cfg/hello_world --output hello_world.bc
+
+          # See https://github.com/lifting-bits/mcsema/issues/753
+          if [ ${{ matrix.host.arch }} == "arm64" ] ; then
+            ./install/bin/mcsema-lift-* --arch x86 --os linux --cfg ../tests/test_suite_generator/generated/prebuilt_cfg/x86/linux/cfg/hello_world --output hello_world.bc || true
+          else
+            ./install/bin/mcsema-lift-* --arch x86 --os linux --cfg ../tests/test_suite_generator/generated/prebuilt_cfg/x86/linux/cfg/hello_world --output hello_world.bc
+          fi
+
       - name: Publish Release Assets
         if: |
           github.event.release

diff --git a/.github/workflows/vcpkg_ci_mac.yml b/.github/workflows/vcpkg_ci_mac.yml
@@ -61,7 +61,7 @@ jobs:
 
       - uses: actions/checkout@v2.3.4
         with:
-          submodules: true
+          submodules: recursive
 
       - uses: actions/setup-dotnet@v1.8.0
         with:
@@ -111,14 +111,14 @@ jobs:
             @dependencies.txt
 
       - name: 'Export Packages'
-        if: contains(github.event.head_commit.message, 'debug artifacts') || github.event.release
+        if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release
         shell: 'bash'
         run: |
           brew install pixz
           ./emit_artifacts.sh ${{ env.ARTIFACT_NAME }}
 
       - uses: actions/upload-artifact@v2.2.3
-        if: "contains(github.event.head_commit.message, 'debug artifacts')"
+        if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.ARTIFACT_NAME }}
@@ -177,7 +177,7 @@ jobs:
             ..
           cmake --build .
           cmake --build . --target install
-          ../scripts/roundtrip.py ./tools/rellic-decomp-* ../tests/tools/decomp "${VCPKG_ROOT}/installed/${TRIPLET}/tools/${{ matrix.llvm }}/clang" || true
+          ../scripts/roundtrip.py ./tools/rellic-decomp-* ../tests/tools/decomp "${VCPKG_ROOT}/installed/${TRIPLET}/tools/llvm/clang"
 
       - name: 'Test remill build'
         shell: 'bash'
@@ -192,6 +192,7 @@ jobs:
           cmake --build .
           cmake --build . --target install
           cmake --build . --target test_dependencies
+          # Sometimes fails on a flaky test
           env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test || true
 
       - name: 'Anvill build - vcpkg remill'
@@ -204,6 +205,7 @@ jobs:
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DVCPKG_ROOT="${VCPKG_ROOT}" \
             -DANVILL_ENABLE_INSTALL_TARGET=ON \
+            -DANVILL_ENABLE_PYTHON3_LIBS=OFF \
             -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
             ..
           cmake --build .
@@ -219,6 +221,25 @@ jobs:
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
+            -DVCPKG_ROOT="${VCPKG_ROOT}" \
+            -DANVILL_ENABLE_INSTALL_TARGET=ON \
+            -DANVILL_ENABLE_PYTHON3_LIBS=OFF \
+            -Dremill_DIR="$(pwd)/../../remill/build/install/lib/cmake/remill" \
+            ..
+          cmake --build .
+          cmake --build . --target install
+          ./install/bin/anvill-decompile-json-* -spec ../tools/decompile-json/tests/specs/ret0.json -bc_out ./ret0.bc -ir_out ret0.ir
+
+      - name: 'Anvill-McSema build - custom remill'
+        shell: 'bash'
+        if: ${{ matrix.llvm != 'llvm-12' }}
+        run: |
+          cd anvill-mcsema
+          mkdir -p build && cd build
+          cmake -G Ninja \
+            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
+            -DVCPKG_ROOT="${VCPKG_ROOT}" \
             -DANVILL_ENABLE_INSTALL_TARGET=ON \
             -Dremill_DIR="$(pwd)/../../remill/build/install/lib/cmake/remill" \
             ..
@@ -228,19 +249,24 @@ jobs:
 
       - name: 'Test mcsema build'
         shell: 'bash'
-        if: ${{ matrix.llvm == 'llvm-11' }}
+        if: ${{ matrix.llvm != 'llvm-12' }}
         run: |
           cd mcsema
           mkdir -p build && cd build
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \
+            -DVCPKG_ROOT="${VCPKG_ROOT}" \
             -Dremill_DIR="$(pwd)/../../remill/build/install/lib/cmake/remill" \
-            -Danvill_DIR="$(pwd)/../../anvill/build/install/lib/cmake/anvill" \
+            -Danvill_DIR="$(pwd)/../../anvill-mcsema/build/install/lib/cmake/anvill" \
             ..
           cmake --build .
           cmake --build . --target install
 
+          ./install/bin/mcsema-lift-* --version
+          ./install/bin/mcsema-lift-* --arch amd64 --os linux --cfg ../tests/test_suite_generator/generated/prebuilt_cfg/amd64/linux/cfg/hello_world --output hello_world.bc
+          ./install/bin/mcsema-lift-* --arch x86 --os linux --cfg ../tests/test_suite_generator/generated/prebuilt_cfg/x86/linux/cfg/hello_world --output hello_world.bc
+
       - name: Publish Release Assets
         if: |
           github.event.release

diff --git a/.github/workflows/vcpkg_ci_windows.yml b/.github/workflows/vcpkg_ci_windows.yml
@@ -61,7 +61,7 @@ jobs:
 
       - uses: actions/checkout@v2.3.4
         with:
-          submodules: true
+          submodules: recursive
 
       - uses: actions/setup-dotnet@v1.8.0
         with:
@@ -200,8 +200,9 @@ jobs:
       #     cd build
       #     cmake -T ClangCl `
       #       -DCMAKE_VERBOSE_MAKEFILE=ON `
+      #       -DVCPKG_ROOT="${env:VCPKG_ROOT}" `
       #       -DANVILL_ENABLE_INSTALL_TARGET=ON `
-      #       -DANVILL_INSTALL_PYTHON3_LIBS=OFF `
+      #       -DANVILL_ENABLE_PYTHON3_LIBS=OFF \
       #       -DVCPKG_ROOT="${env:VCPKG_ROOT}" `
       #       -DCMAKE_INSTALL_PREFIX="$(pwd)\install" `
       #       ..
@@ -219,8 +220,28 @@ jobs:
           cmake -T ClangCl `
             -DCMAKE_VERBOSE_MAKEFILE=ON `
             -DCMAKE_INSTALL_PREFIX="$(pwd)\install" `
+            -DVCPKG_ROOT="${env:VCPKG_ROOT}" `
+            -DANVILL_ENABLE_INSTALL_TARGET=ON `
+            -DANVILL_ENABLE_PYTHON3_LIBS=OFF `
+            -Dremill_DIR="$(pwd)\..\..\remill\build\install\lib\cmake\remill" `
+            ..
+          cmake --build . --config Release -j
+          cmake --build . --config Release --target install
+
+          & ( Get-ChildItem install\bin | Where-Object {$_.name -match "anvill-decompile-json-.*.exe"} ) -spec ..\tools\decompile-json\tests\specs\ret0.json -bc_out ret0.bc -ir_out ret0.ir
+
+      - name: 'Anvill-McSema build - custom remill'
+        if: ${{ matrix.llvm == 'llvm-11' }}
+        run: |
+          cd anvill-mcsema
+          Remove-Item -Recurse -Force -ErrorAction SilentlyContinue .\build
+          New-Item -Path .\build -ItemType Directory
+          cd build
+          cmake -T ClangCl `
+            -DCMAKE_VERBOSE_MAKEFILE=ON `
+            -DCMAKE_INSTALL_PREFIX="$(pwd)\install" `
+            -DVCPKG_ROOT="${env:VCPKG_ROOT}" `
             -DANVILL_ENABLE_INSTALL_TARGET=ON `
-            -DANVILL_INSTALL_PYTHON3_LIBS=OFF `
             -Dremill_DIR="$(pwd)\..\..\remill\build\install\lib\cmake\remill" `
             ..
           cmake --build . --config Release -j
@@ -238,14 +259,19 @@ jobs:
           cmake -T ClangCl `
             -DCMAKE_VERBOSE_MAKEFILE=ON `
             -DCMAKE_INSTALL_PREFIX="$(pwd)\install" `
+            -DVCPKG_ROOT="${env:VCPKG_ROOT}" `
             -DMCSEMA_INSTALL_PYTHON2_LIBS=OFF `
             -DMCSEMA_INSTALL_PYTHON3_LIBS=OFF `
             -Dremill_DIR="$(pwd)\..\..\remill\build\install\lib\cmake\remill" `
-            -Danvill_DIR="$(pwd)\..\..\anvill\build\install\lib\cmake\anvill" `
+            -Danvill_DIR="$(pwd)\..\..\anvill-mcsema\build\install\lib\cmake\anvill" `
             ..
           cmake --build . --config Release -j
           cmake --build . --config Release --target install
 
+          & ( Get-ChildItem install\mcsema\bin | Where-Object {$_.name -match "mcsema-lift-.*.exe"} ) --version
+          & ( Get-ChildItem install\mcsema\bin | Where-Object {$_.name -match "mcsema-lift-.*.exe"} ) --arch amd64 --os linux --cfg ..\tests\test_suite_generator\generated\prebuilt_cfg\amd64\linux\cfg\hello_world --output hello_world.bc
+          & ( Get-ChildItem install\mcsema\bin | Where-Object {$_.name -match "mcsema-lift-.*.exe"} ) --arch x86 --os linux --cfg ..\tests\test_suite_generator\generated\prebuilt_cfg\x86\linux\cfg\hello_world --output hello_world.bc
+
       - name: Publish Release Assets
         if: |
           github.event.release

diff --git a/.gitmodules b/.gitmodules
@@ -14,3 +14,6 @@
 	path = rellic
 	url = https://github.com/lifting-bits/rellic.git
 	branch = master
+[submodule "anvill-mcsema"]
+	path = anvill-mcsema
+	url = https://github.com/lifting-bits/anvill.git
diff --git a/anvill b/anvill
diff --git a/anvill-mcsema b/anvill-mcsema
diff --git a/mcsema b/mcsema
diff --git a/ports/remill/portfile.cmake b/ports/remill/portfile.cmake
@@ -1,9 +1,8 @@
 vcpkg_from_github(
     OUT_SOURCE_PATH SOURCE_PATH
     REPO lifting-bits/remill
-    # Latest commit after v4.0.22 at this time
-    REF 021b799efc07a38d5f910bb41bdb7e60e4406f23
-    SHA512 10a33d785be69bda54f08ab9f89ccfd9e71bab632d90541eef2ce1122b1161b690cfb5068ee4ae036152a5ebefe8d53dde7f8bb5f99310f678365e9962a417ff
+    REF v4.0.24
+    SHA512 bd1cd52a45174a9f033a490277974fb303dfa8cc4823a51366e9afc31e57d73fb787da98af7efe22675095214a90d14d644714756b8808beec5eb2c648276f82
     HEAD_REF master
 )
 

diff --git a/ports/remill/vcpkg.json b/ports/remill/vcpkg.json
@@ -1,6 +1,6 @@
 {
   "name": "remill",
-  "version-string": "4.0.22",
+  "version-string": "4.0.24",
   "description": "Library for lifting of x86, amd64, and aarch64 machine code to LLVM bitcode",
   "homepage": "https://github.com/lifting-bits/remill",
   "default-features": [
@@ -24,9 +24,17 @@
       "dependencies": [
           "llvm-11"
       ]
+    },
+    "llvm-12": {
+      "description": "Use LLVM 12",
+      "dependencies": [
+          "llvm-12"
+      ]
     }
   },
   "dependencies": [
+    "glog",
+    "gflags",
     "xed"
   ]
 }
diff --git a/rellic b/rellic
diff --git a/remill b/remill