perf: replace CFFI with a native C extension (#76)

Closes #68.

.appveyor.yml

@@ -44,6 +44,7 @@ build_script:
     - cd %BUILD_DIR%
     - cmake
         -G "%CMAKE_GENERATOR%"
+        -DCMAKE_BUILD_TYPE=Release
         -DCRC32C_BUILD_TESTS=no
         -DCRC32C_BUILD_BENCHMARKS=no
         -DBUILD_SHARED_LIBS=yes

.github/workflows/presubmit.yml

@@ -45,9 +45,8 @@ jobs:
         ./scripts/manylinux/build.sh
     - name: Test Import
       run: |
-        pip install cffi pycparser
         pip install --no-index --find-links=wheels google-crc32c
-        python ./scripts/check_cffi_crc32c.py
+        python ./scripts/check_crc32c_extension.py
     - name: Run tests
       run: |
         pip install pytest
@@ -81,9 +80,8 @@ jobs:
         ./scripts/osx/build_gh_action.sh
     - name: Test Import
       run: |
-        pip install cffi pycparser
         pip install --no-index --find-links=wheels google-crc32c
-        python ./scripts/check_cffi_crc32c.py
+        python ./scripts/check_crc32c_extension.py
     - name: Run tests
       run: |
         pip install pytest

.github/workflows/python-publish.yml

@@ -41,9 +41,8 @@ jobs:
         ./scripts/manylinux/build.sh
     - name: Test Import
       run: |
-        pip install cffi pycparser
         pip install --no-index --find-links=wheels google-crc32c
-        python ./scripts/check_cffi_crc32c.py
+        python ./scripts/check_crc32c_extension.py
     - name: Run tests
       run: |
         pip install pytest
@@ -73,9 +72,8 @@ jobs:
         ./scripts/osx/build_gh_action.sh
     - name: Test Import
       run: |
-        pip install cffi pycparser
         pip install --no-index --find-links=wheels google-crc32c
-        python ./scripts/check_cffi_crc32c.py
+        python ./scripts/check_crc32c_extension.py
     - name: Run tests
       run: |
         pip install pytest

BUILDING.md

@@ -19,11 +19,10 @@ On Linux:
 
 
 # Install the wheel that was built as a result
-pip install cffi pycparser
 pip install --no-index --find-links=wheels google-crc32c
 
 # Check the package, try and load the native library.
-python ./scripts/check_cffi_crc32c.py
+python ./scripts/check_crc32c_extension.py
 ```
 
 On OS X:
@@ -33,11 +32,10 @@ On OS X:
 ./scripts/osx/build.sh
 
 # Install the wheel that was built as a result
-pip install cffi pycparser
 pip install --no-index --find-links=wheels google-crc32c
 
 # Check the package, try and load the native library.
-python ./scripts/check_cffi_crc32c.py
+python ./scripts/check_crc32c_extension.py
 ```
 
 On Windows:
@@ -64,6 +62,7 @@ $ export CRC32C_INSTALL_PREFIX=$(pwd)/usr
 $ mkdir google_crc32c/build
 $ cd google_crc32c/build
 $ ../../venv/bin/cmake \
+>   -DCMAKE_BUILD_TYPE=Release \
 >   -DCRC32C_BUILD_TESTS=no \
 >   -DCRC32C_BUILD_BENCHMARKS=no \
 >   -DBUILD_SHARED_LIBS=yes \
@@ -76,6 +75,10 @@ $ cd ../..
 Now, run the tests:
 
 ```bash
+$ venv/bin/python setup.py build_ext \
+    --include-dirs=$(pwd)/usr/include \
+    --library-dirs=$(pwd)/usr/lib \
+    --rpath=$(pwd)/usr/lib
 $ venv/bin/pip install -e .[testing]
 $ venv/bin/py.test tests/
 ============================= test session starts ==============================

noxfile.py

@@ -16,27 +16,34 @@
 
 # Generated by synthtool. DO NOT EDIT!
 
-from __future__ import absolute_import
+import os
+import sys
+
 import nox
-from sys import platform
 
 
+HERE = os.path.dirname(__file__)
+
 @nox.session(python=["3.6", "3.7", "3.8", "3.9"])
 def build_libcrc32c(session):
-    if platform.startswith("win"):
+    session.env["PY_BIN"] = f"python{session.python}"
+    session.env["REPO_ROOT"] = HERE
+
+    if sys.platform.startswith("win"):
         session.run("cmd", "-c", "scripts\\windows\\build.bat")
-    elif platform == "linux":
+    elif sys.platform == "linux":
         session.run("bash", "scripts/local-linux/build_libcrc32c.sh")
-    elif platform == "darwin":
+    elif sys.platform == "darwin":
         session.run("bash", "scripts/osx/build.sh")
     else:
         raise Exception("Unsupported")
 
 
 @nox.session(python=["3.6", "3.7", "3.8", "3.9"])
 def check(session):
-    session.install("cffi>=1.0.0")
-    session.install("--no-index", "--find-links=wheels", "google-crc32c")
+    session.install("pytest")
+    session.install("--no-index", f"--find-links={HERE}/wheels", "google-crc32c")
 
     # Run py.test against the unit tests.
-    session.run("python", "scripts/check_cffi_crc32c.py", *session.posargs)
+    session.run("py.test", "tests")
+    session.run("python", f"{HERE}/scripts/check_crc32c_extension.py", *session.posargs)

scripts/check_cffi_crc32c.py → scripts/check_crc32c_extension.py

@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from google_crc32c import _crc32c_cffi
+from google_crc32c import _crc32c
 
 
 def main():
-    print("_crc32c_cffi: {}".format(_crc32c_cffi))
-    print("_crc32c_cffi.lib: {}".format(_crc32c_cffi.lib))
-    print("dir(_crc32c_cffi.lib): {}".format(dir(_crc32c_cffi.lib)))
+    print("_crc32c: {}".format(_crc32c))
+    print("dir(_crc32c): {}".format(dir(_crc32c)))
 
 
 if __name__ == "__main__":

scripts/dev-requirements.txt

@@ -1,5 +1,4 @@
 auditwheel >= 2.1.1; sys_platform == 'linux' or sys_platform == 'linux2'
-cffi >= 1.13.2
 delocate >= 0.8.0; sys_platform == 'darwin'
 setuptools >= 42.0.2
 # See: https://github.com/pypa/auditwheel/issues/102

scripts/local-linux/build_libcrc32c.sh

@@ -14,26 +14,26 @@
 
 set -e -x
 
-PY_BIN="python3.7"
-
-# Check that the REPO_ROOT variable is set.
-if [[ -z "${REPO_ROOT}" ]]; then
-    echo "REPO_ROOT environment variable should be set by the caller."
-    exit 1
-fi
+PY_BIN=${PY_BIN:-python3.7}
+REPO_ROOT=${REPO_ROOT:-$(pwd)}
 
 CRC32C_INSTALL_PREFIX=${REPO_ROOT}/usr
 
 # NOTE: This assumes the local install has an up-to-date `pip`.
 # Create a virtualenv where we can install `cmake`.
 VENV=${REPO_ROOT}/venv
 ${PY_BIN} -m venv ${VENV}
+${VENV}/bin/python -m pip install --upgrade setuptools pip wheel
 ${VENV}/bin/python -m pip install "cmake >= 3.12.0"
+rm -rf ${REPO_ROOT}/build
+rm -rf ${CRC32C_INSTALL_PREFIX}
 # Build `libcrc32c`
 cd ${REPO_ROOT}/google_crc32c
+rm -rf build
 mkdir build
 cd build/
 ${VENV}/bin/cmake \
+    -DCMAKE_BUILD_TYPE=Release \
     -DCRC32C_BUILD_TESTS=no \
     -DCRC32C_BUILD_BENCHMARKS=no \
     -DBUILD_SHARED_LIBS=yes \
@@ -42,6 +42,14 @@ ${VENV}/bin/cmake \
 # Install `libcrc32c` into CRC32C_INSTALL_PREFIX.
 make all install
 
+cd ${REPO_ROOT}
+
+${VENV}/bin/python setup.py build_ext \
+    --include-dirs=${REPO_ROOT}/usr/include \
+    --library-dirs=${REPO_ROOT}/usr/lib \
+    --rpath=${REPO_ROOT}/usr/lib
+${VENV}/bin/python -m pip wheel . --wheel-dir=wheels
+
 # Clean up.
 rm -fr ${REPO_ROOT}/google_crc32c/build
 rm -fr ${VENV}

scripts/manylinux/build_on_centos.sh

@@ -17,20 +17,23 @@ set -e -x
 
 MAIN_PYTHON_BIN="/opt/python/cp37-cp37m/bin"
 echo "BUILD_PYTHON: ${BUILD_PYTHON}"
+REPO_ROOT=/var/code/python-crc32c/
 
 # Upgrade `pip` before using it.
 ${MAIN_PYTHON_BIN}/python -m pip install --upgrade pip
 # Install `cmake` (i.e. non-Python build dependency).
 ${MAIN_PYTHON_BIN}/python -m pip install "cmake >= 3.12.0"
 # Install Python build dependencies.
 ${MAIN_PYTHON_BIN}/python -m pip install \
-    --requirement /var/code/python-crc32c/scripts/dev-requirements.txt
+    --requirement ${REPO_ROOT}/scripts/dev-requirements.txt
 
 # Build and install `crc32c`
-cd /var/code/python-crc32c/google_crc32c/
+cd ${REPO_ROOT}/google_crc32c/
+rm -rf build
 mkdir build
 cd build/
 ${MAIN_PYTHON_BIN}/cmake \
+    -DCMAKE_BUILD_TYPE=Release \
     -DCRC32C_BUILD_TESTS=no \
     -DCRC32C_BUILD_BENCHMARKS=no \
     -DBUILD_SHARED_LIBS=yes \
@@ -72,11 +75,11 @@ else
 fi
 
 # Build the wheels.
-cd /var/code/python-crc32c/
+cd ${REPO_ROOT}
 for PYTHON_BIN in ${VERSION_WHITELIST}; do
     ${PYTHON_BIN}/python -m pip install --upgrade pip
     ${PYTHON_BIN}/python -m pip install \
-        --requirement /var/code/python-crc32c/scripts/dev-requirements.txt
+        --requirement ${REPO_ROOT}/scripts/dev-requirements.txt
     ${PYTHON_BIN}/python -m pip wheel . --wheel-dir dist_wheels/
 done
 
@@ -86,5 +89,5 @@ for whl in dist_wheels/google_crc32c*.whl; do
 done
 
 # Clean up.
-rm -fr /var/code/python-crc32c/google_crc32c/build/
-rm -fr /var/code/python-crc32c/dist_wheels/
+rm -fr ${REPO_ROOT}/google_crc32c/build/
+rm -fr ${REPO_ROOT}/dist_wheels/

scripts/manylinux/check.sh

@@ -37,7 +37,7 @@ ${PYTHON} -m venv venv
 venv/bin/pip install ${WHEEL_FILE}
 
 # Verify that the module is installed and peek at contents.
-venv/bin/python scripts/check_cffi_crc32c.py
+venv/bin/python scripts/check_crc32c_extension.py
 unzip -l ${WHEEL_FILE}
 
 # Clean up.

scripts/osx/build.sh

@@ -30,7 +30,7 @@ PYTHON38="/Library/Frameworks/Python.framework/Versions/3.8/bin"
 PYTHON39="/Library/Frameworks/Python.framework/Versions/3.9/bin"
 
 # Build and install `libcrc32c`
-export PY_BIN="python3"
+export PY_BIN="${PY_BIN:-python3}"
 export CRC32C_INSTALL_PREFIX="${REPO_ROOT}/usr"
 
 cd ${REPO_ROOT}

scripts/osx/build_c_lib.sh

@@ -58,6 +58,7 @@ ls
 
 # We don't build i386 anymore as XCode no longer supports.
 ${VENV}/bin/cmake \
+    -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_OSX_DEPLOYMENT_TARGET=10.9 \
     -DCMAKE_OSX_ARCHITECTURES="x86_64" \
     -DCRC32C_BUILD_TESTS=no \

scripts/osx/build_python_wheel.sh

@@ -44,6 +44,10 @@ ${VENV}/bin/python -m pip install \
 DIST_WHEELS="${REPO_ROOT}/dist_wheels"
 mkdir -p ${DIST_WHEELS}
 cd ${REPO_ROOT}
+${VENV}/bin/python setup.py build_ext \
+    --include-dirs=${REPO_ROOT}/usr/include \
+    --library-dirs=${REPO_ROOT}/usr/lib \
+    --rpath=${REPO_ROOT}/usr/lib
 ${VENV}/bin/python -m pip wheel ${REPO_ROOT} --wheel-dir ${DIST_WHEELS}
 
 # Delocate the wheel. removed --check-archs. We don't build i386.

scripts/osx/check.sh

@@ -49,23 +49,29 @@ ${PYTHON36}/python3 -m venv venv36
 curl https://bootstrap.pypa.io/get-pip.py | venv36/bin/python3
 WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp36-cp36m-macosx_10_9_x86_64.whl
 venv36/bin/pip install ${WHL}
-venv36/bin/python ${REPO_ROOT}/scripts/check_cffi_crc32c.py
+venv36/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py
+venv36/bin/pip install pytest
+venv36/bin/py.test ${REPO_ROOT}/tests
 ${LISTDEPS_CMD} ${WHL}
 rm -fr venv36
 
 ${PYTHON37}/python3 -m venv venv37
 curl https://bootstrap.pypa.io/get-pip.py | venv37/bin/python3
 WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp37-cp37m-macosx_10_9_x86_64.whl
 venv37/bin/pip install ${WHL}
-venv37/bin/python ${REPO_ROOT}/scripts/check_cffi_crc32c.py
+venv37/bin/pip install pytest
+venv37/bin/py.test ${REPO_ROOT}/tests
+venv37/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py
 ${LISTDEPS_CMD} ${WHL}
 rm -fr venv37
 
 ${PYTHON38}/python3 -m venv venv38
 curl https://bootstrap.pypa.io/get-pip.py | venv38/bin/python3
 WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp38-cp38-macosx_10_9_x86_64.whl
 venv38/bin/pip install ${WHL}
-venv38/bin/python ${REPO_ROOT}/scripts/check_cffi_crc32c.py
+venv38/bin/pip install pytest
+venv38/bin/py.test ${REPO_ROOT}/tests
+venv38/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py
 ${LISTDEPS_CMD} ${WHL}
 rm -fr venv38
 
@@ -74,6 +80,6 @@ rm -fr venv38
 # curl https://bootstrap.pypa.io/get-pip.py | venv39/bin/python3
 # WHL=${REPO_ROOT}/wheels/google_crc32c-${PACKAGE_VERSION}-cp39-cp39-macosx_10_9_x86_64.whl
 # venv37/bin/pip install ${WHL}
-# venv37/bin/python ${REPO_ROOT}/scripts/check_cffi_crc32c.py
+# venv37/bin/python ${REPO_ROOT}/scripts/check_crc32c_extension.py
 # ${LISTDEPS_CMD} ${WHL}
 # rm -fr venv39

scripts/windows/build.bat

@@ -39,7 +39,7 @@ py -%PYTHON_VERSION% -m pip install cmake
 
 git submodule update --init --recursive
 
-FOR %%V IN (32,64) DO (ddd
+FOR %%V IN (32,64) DO (
     set TARGET_PLATFORM="x64"
 
     if "%%V"=="32" (
@@ -61,7 +61,7 @@ FOR %%V IN (32,64) DO (ddd
 
     echo "Running cmake with Generator:  %CMAKE_GENERATOR%, Platform: !TARGET_PLATFORM!, Install Prefix: %CRC32C_INSTALL_PREFIX%"
 
-    %cmake% -G %CMAKE_GENERATOR% -A !TARGET_PLATFORM! -DCRC32C_BUILD_BENCHMARKS=no -DCRC32C_BUILD_TESTS=no -DBUILD_SHARED_LIBS=yes -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=yes -DCRC32C_USE_GLOG=0 -DCMAKE_INSTALL_PREFIX:PATH=%CRC32C_INSTALL_PREFIX% ..
+    %cmake% -G %CMAKE_GENERATOR% -A !TARGET_PLATFORM! -DCMAKE_BUILD_TYPE=Release -DCRC32C_BUILD_BENCHMARKS=no -DCRC32C_BUILD_TESTS=no -DBUILD_SHARED_LIBS=yes -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=yes -DCRC32C_USE_GLOG=0 -DCMAKE_INSTALL_PREFIX:PATH=%CRC32C_INSTALL_PREFIX% ..
 
     %cmake% --build . --config "%CONFIGURATION%" --target install
 
@@ -73,5 +73,8 @@ FOR %%V IN (32,64) DO (ddd
     copy %CRC32C_INSTALL_PREFIX%\bin\crc32c.dll .
 
     py -%PYTHON_VERSION%-%%V -m pip install --upgrade pip setuptools wheel
+    echo "Building C extension"
+    py -%PYTHON_VERSION%-%%V setup.py build_ext --include-dirs=%CRC32C_INSTALL_PREFIX%\include --library-dirs=%CRC32C_INSTALL_PREFIX%\lib
+    echo "Building Wheel"
     py -%PYTHON_VERSION%-%%V -m pip wheel . --wheel-dir wheels/
 )

scripts/windows/test.bat

@@ -27,11 +27,10 @@ if "%PYTHON_VERSION%"=="3.10-dev" (
 @rem FOR %%V IN (3.5-64,3.5-32,3.6-64,3.6-32,3.7-64,3.7-32) DO (
 @REM FOR %%V IN (3.9-64,3.9-32) DO (
 FOR %%V IN (%PYTHON_VERSION%-32, %PYTHON_VERSION%-64) DO (
-    py -%%V -m pip install cffi pyparser
     py -%%V -m pip install --no-index --find-links=wheels google-crc32c --force-reinstall
 
-    py -%%V ./scripts/check_cffi_crc32c.py
+    py -%%V ./scripts/check_crc32c_extension.py
 
     py -%%V -m pip install pytest
     py -%%V -m pytest tests
-)
+)