[vpdq] Add multithreaded hashing (#1338)

facebook · Jul 24, 2023 · c1dcb01 · c1dcb01
1 parent ad84a89
commit c1dcb01
Show file tree

Hide file tree

Showing 9 changed files with 585 additions and 393 deletions.
diff --git a/.github/workflows/vpdq-ci-cpp.yml b/.github/workflows/vpdq-ci-cpp.yml
@@ -27,15 +27,31 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: "3.8"
-      - name: make
+      - name: Install dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y make cmake ffmpeg libavdevice-dev libavfilter-dev libavformat-dev libavcodec-dev libswresample-dev libswscale-dev libavutil-dev
-          mkdir -p cpp/build
+
+      - name: Make with address,leak,undefined sanitizer
+        run: |
+          rm -rf cpp/build
+          mkdir cpp/build
           cd cpp/build
-          cmake ..
+          cmake .. -DCMAKE_CXX_FLAGS="-O2 -fPIC -Wall -Wextra -Werror -Wno-unused-function -Wno-deprecated-declarations -fsanitize=address,leak,undefined" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread,undefined"
           make
-      - name: regression test
+
+      - name: Test with address,leak,undefined sanitizer
+        run: |
+          python cpp/regtest.py -i ${GITHUB_WORKSPACE}/tmk/sample-videos
+
+      - name: Make with thread,undefined sanitizer
+        run: |
+          rm -rf cpp/build
+          mkdir cpp/build
+          cd cpp/build
+          cmake .. -DCMAKE_CXX_FLAGS="-O2 -fPIC -Wall -Wextra -Werror -Wno-unused-function -Wno-deprecated-declarations -fsanitize=thread,undefined" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,leak,undefined"
+          make
+
+      - name: Test with thread,undefined sanitizer
         run: |
-          mkdir -p cpp/output-hashes
-          python cpp/regtest.py -d ${GITHUB_WORKSPACE}/vpdq/cpp/output-hashes -i ${GITHUB_WORKSPACE}/tmk/sample-videos
+          python cpp/regtest.py -i ${GITHUB_WORKSPACE}/tmk/sample-videos
diff --git a/.github/workflows/vpdq-ci-python.yaml b/.github/workflows/vpdq-ci-python.yaml
@@ -7,12 +7,14 @@ on:
       - main
     paths:
       - "vpdq/python/**"
+      - "vpdq/cpp/**"
       - ".github/workflows/vpdq-ci-python.yaml"
   pull_request:
     branches:
       - main
     paths:
       - "vpdq/python/**"
+      - "vpdq/cpp/**"
       - ".github/workflows/vpdq-ci-python.yaml"
 
 defaults:
@@ -44,4 +46,4 @@ jobs:
           python3 vpdq-release.py -i
       - name: Test with pytest
         run: |
-          py.test
+          python -m pytest
diff --git a/pdq/cpp/hashing/pdqhashing.cpp b/pdq/cpp/hashing/pdqhashing.cpp
@@ -2,6 +2,8 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
 // ================================================================
 
+#include <mutex>
+
 #include <pdq/cpp/downscaling/downscaling.h>
 #include <pdq/cpp/hashing/pdqhashing.h>
 #include <pdq/cpp/hashing/torben.h>
@@ -531,18 +533,18 @@ void pdqBuffer16x16ToBits(float dctOutput16x16[16][16], Hash256* hashptr) {
 // * Storage is row-major
 // * Element i,j at row i column j is at offset i*16+j.
 static float* fill_dct_matrix_64_cached() {
-  static bool initialized = false;
+  static std::once_flag initialized;
   static float buffer[16 * 64];
-  if (!initialized) {
+
+  std::call_once(initialized, []() {
     const float matrix_scale_factor = std::sqrt(2.0 / 64.0);
     for (int i = 0; i < 16; i++) {
       for (int j = 0; j < 64; j++) {
         buffer[i * 64 + j] = matrix_scale_factor *
             cos((M_PI / 2 / 64.0) * (i + 1) * (2 * j + 1));
       }
     }
-    initialized = false;
-  }
+  });
   return &buffer[0];
 }
 

diff --git a/vpdq/cpp/CMakeLists.txt b/vpdq/cpp/CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.17)
 
-set(CMAKE_CXX_FLAGS "-O2 -fPIC -Wall -Wextra -Werror -Wno-unused-function")
+set(CMAKE_CXX_FLAGS "-O2 -fPIC -Wall -Wextra -Werror -Wno-unused-function -Wno-deprecated-declarations")
 project(VPDQ)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(LIBAV REQUIRED IMPORTED_TARGET

diff --git a/vpdq/cpp/bin/vpdq-hash-video.cpp b/vpdq/cpp/bin/vpdq-hash-video.cpp
@@ -76,6 +76,7 @@ int main(int argc, char** argv) {
   std::string outputDirectory = "";
   double secondsPerHash = 0;
   int downsampleFrameDimension = 0;
+  unsigned int thread_count = 0;
 
   while ((argi < argc) && argv[argi][0] == '-') {
     std::string flag(argv[argi++]);
@@ -125,6 +126,13 @@ int main(int argc, char** argv) {
       downsampleFrameDimension = std::atoi(argv[argi++]);
       continue;
     }
+    if (flag == "-t" || flag == "--thread-count") {
+      if ((argc - argi) < 1) {
+        usage(argv[0], 1);
+      }
+      thread_count = std::atoi(argv[argi++]);
+      continue;
+    }
     usage(argv[0], 1);
   }
 
@@ -167,7 +175,8 @@ int main(int argc, char** argv) {
       verbose,
       secondsPerHash,
       downsampleFrameDimension,
-      downsampleFrameDimension);
+      downsampleFrameDimension,
+      thread_count);
   if (!rc) {
     fprintf(
         stderr,