Add BLAKE3 hashing algorithm (single-threaded C-based implementation)

NixOS · Apr 23, 2024 · a1b115c · a1b115c
1 parent e3a4e40
commit a1b115c
Show file tree

Hide file tree

Showing 8 changed files with 120 additions and 9 deletions.
diff --git a/flake.nix b/flake.nix
@@ -181,6 +181,26 @@
 
           changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { };
 
+          blake3-src-nix = final.stdenv.mkDerivation rec {
+            pname = "blake3-src-nix";
+            version = "1.5.1";
+            src = final.fetchzip {
+              url = "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/${version}.tar.gz";
+              hash = "sha512-fEdcjMJiqFkeXL91L95kmvl8fTo8LrGnFuT5NLLa+zFkDo4OalpepHTbi/goD1m8aNVzGwhmfnscOzeKv+gMRg==";
+            };
+            phases = [ "unpackPhase" "installPhase" ];
+            installPhase = ''
+              mkdir -p $out
+              cp -a ${src}/* $out
+            '';
+            meta = {
+              description = "The BLAKE3 source code";
+              homepage = "https://github.com/BLAKE3-team/BLAKE3";
+              license = lib.licenses.asl20;
+              platforms = lib.platforms.all;
+            };
+          };
+
           nix =
             let
               officialRelease = false;
@@ -199,6 +219,7 @@
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
+              blake3-src = final.blake3-src-nix;
             } // {
               # this is a proper separate downstream package, but put
               # here also for back compat reasons.

diff --git a/package.nix b/package.nix
@@ -6,6 +6,7 @@
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
+, blake3-src
 , buildPackages
 , nlohmann_json
 , bison
@@ -203,6 +204,9 @@ in {
 
   VERSION_SUFFIX = versionSuffix;
 
+  # Export the BLAKE3 source location so we can access it from the make files.
+  NIX_BLAKE3_SRC = blake3-src;
+
   outputs = [ "out" ]
     ++ lib.optional doBuild "dev"
     # If we are doing just build or just docs, the one thing will use
@@ -233,6 +237,7 @@ in {
   ;
 
   buildInputs = lib.optionals doBuild [
+    blake3-src
     boost
     brotli
     bzip2

diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc
@@ -50,7 +50,7 @@ Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
 {
     return Args::Flag {
             .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
+            .description = "Hash algorithm (`blake3`, `md5`, `sha1`, `sha256`, or `sha512`).",
             .labels = {"hash-algo"},
             .handler = {[ha](std::string s) {
                 *ha = parseHashAlgo(s);
@@ -63,7 +63,7 @@ Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * o
 {
     return Args::Flag {
             .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
+            .description = "Hash algorithm (`blake3`, `md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
             .labels = {"hash-algo"},
             .handler = {[oha](std::string s) {
                 *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
@@ -4,6 +4,7 @@
 #include <openssl/crypto.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
+#include "blake3.h"
 
 #include "args.hh"
 #include "hash.hh"
@@ -20,6 +21,7 @@ namespace nix {
 
 static size_t regularHashSize(HashAlgorithm type) {
     switch (type) {
+    case HashAlgorithm::BLAKE3: return blake3HashSize;
     case HashAlgorithm::MD5: return md5HashSize;
     case HashAlgorithm::SHA1: return sha1HashSize;
     case HashAlgorithm::SHA256: return sha256HashSize;
@@ -29,7 +31,7 @@ static size_t regularHashSize(HashAlgorithm type) {
 }
 
 
-const std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
+const std::set<std::string> hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" };
 
 const std::set<std::string> hashFormats = {"base64", "nix32", "base16", "sri" };
 
@@ -282,9 +284,11 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha
         return Hash::parseAny(hashStr, ha);
 }
 
+typedef blake3_hasher BLAKE3_CTX;
 
 union Ctx
 {
+    BLAKE3_CTX blake3;
     MD5_CTX md5;
     SHA_CTX sha1;
     SHA256_CTX sha256;
@@ -294,7 +298,8 @@ union Ctx
 
 static void start(HashAlgorithm ha, Ctx & ctx)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_init(&ctx.blake3);
+    else if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
     else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1);
     else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256);
     else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
@@ -304,7 +309,8 @@ static void start(HashAlgorithm ha, Ctx & ctx)
 static void update(HashAlgorithm ha, Ctx & ctx,
                    std::string_view data)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size());
+    else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
@@ -313,7 +319,8 @@ static void update(HashAlgorithm ha, Ctx & ctx,
 
 static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_finalize(&ctx.blake3, hash, BLAKE3_OUT_LEN);
+    else if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
     else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1);
     else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256);
     else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512);
@@ -427,6 +434,7 @@ std::string_view printHashFormat(HashFormat HashFormat)
 
 std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
 {
+    if (s == "blake3") return HashAlgorithm::BLAKE3;
     if (s == "md5") return HashAlgorithm::MD5;
     if (s == "sha1") return HashAlgorithm::SHA1;
     if (s == "sha256") return HashAlgorithm::SHA256;
@@ -440,12 +448,13 @@ HashAlgorithm parseHashAlgo(std::string_view s)
     if (opt_h)
         return *opt_h;
     else
-        throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s);
+        throw UsageError("unknown hash algorithm '%1%', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", s);
 }
 
 std::string_view printHashAlgo(HashAlgorithm ha)
 {
     switch (ha) {
+    case HashAlgorithm::BLAKE3: return "blake3";
     case HashAlgorithm::MD5: return "md5";
     case HashAlgorithm::SHA1: return "sha1";
     case HashAlgorithm::SHA256: return "sha256";

diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
@@ -11,9 +11,10 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 };
+enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512, BLAKE3 };
 
 
+const int blake3HashSize = 32;
 const int md5HashSize = 16;
 const int sha1HashSize = 20;
 const int sha256HashSize = 32;

diff --git a/src/libutil/local.mk b/src/libutil/local.mk
@@ -40,3 +40,64 @@ $(foreach i, $(wildcard $(d)/signature/*.hh), \
 ifeq ($(HAVE_LIBCPUID), 1)
   libutil_LDFLAGS += -lcpuid
 endif
+
+# BLAKE3 support
+
+# See the comments below for the build rules for the reason we use the `.o` files here.
+
+libutil_SOURCES += \
+  $(libutil_DIR)/blake3.o \
+  $(libutil_DIR)/blake3_dispatch.o \
+  $(libutil_DIR)/blake3_portable.o
+
+ifneq ($(filter aarch64% arm%,$(system)),)
+  libutil_SOURCES += $(libutil_DIR)/blake3_neon.o
+endif
+
+ifneq ($(filter x86_64%,$(system)),)
+  ifneq ($(or $(HOST_UNIX), $(HOST_LINUX)),)
+    libutil_SOURCES += \
+      $(libutil_DIR)/blake3_sse2_x86-64_unix.o \
+      $(libutil_DIR)/blake3_sse41_x86-64_unix.o \
+      $(libutil_DIR)/blake3_avx2_x86-64_unix.o \
+      $(libutil_DIR)/blake3_avx512_x86-64_unix.o
+  endif
+  ifdef HOST_WINDOWS
+    ifneq ($(or $(HOST_CYGWIN), $(HOST_MINGW)),)
+      libutil_SOURCES += \
+        $(libutil_DIR)/blake3_sse2_x86-64_windows_gnu.o \
+        $(libutil_DIR)/blake3_sse41_x86-64_windows_gnu.o \
+        $(libutil_DIR)/blake3_avx2_x86-64_windows_gnu.o \
+        $(libutil_DIR)/blake3_avx512_x86-64_windows_gnu.o
+    else
+      libutil_SOURCES += \
+        $(libutil_DIR)/blake3_sse2_x86-64_windows_mscv.o \
+        $(libutil_DIR)/blake3_sse41_x86-64_windows_mscv.o \
+        $(libutil_DIR)/blake3_avx2_x86-64_windows_mscv.o \
+        $(libutil_DIR)/blake3_avx512_x86-64_windows_mscv.o
+    endif
+  endif
+endif
+
+INCLUDE_libutil += -I $(NIX_BLAKE3_SRC)/c
+
+# We use custom rules for the BLAKE3 source files because they are in a read-only store directory
+# and the default rules will try write build artifacts there and fail with permission errors.
+
+$(libutil_DIR)/blake3%.o: $(NIX_BLAKE3_SRC)/c/blake3%.S
+	$(trace-cc) $(call CC_CMD,$@)
+
+$(libutil_DIR)/blake3%.o: $(NIX_BLAKE3_SRC)/c/blake3%.c
+	$(trace-cc) $(call CC_CMD,$@)
+
+$(libutil_DIR)/blake3.o: $(NIX_BLAKE3_SRC)/c/blake3.c
+	$(trace-cc) $(call CC_CMD,$@)
+
+$(libutil_DIR)/blake3%.compile_commands.json: $(NIX_BLAKE3_SRC)/c/blake3%.S
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@
+
+$(libutil_DIR)/blake3%.compile_commands.json: $(NIX_BLAKE3_SRC)/c/blake3%.c
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@
+
+$(libutil_DIR)/blake3.compile_commands.json: $(NIX_BLAKE3_SRC)/c/blake3.c
+	$(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@
diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc
@@ -1084,7 +1084,7 @@ namespace nix {
 
         ASSERT_TRACE1("hashString \"foo\" \"content\"",
                       UsageError,
-                      HintFmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
+                      HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
 
         ASSERT_TRACE2("hashString \"sha256\" {}",
                       TypeError,

diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc
@@ -10,6 +10,20 @@ namespace nix {
      * hashString
      * --------------------------------------------------------------------------*/
 
+    TEST(hashString, testKnownBLAKE3Hashes1) {
+        auto s = "abc";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85");
+    }
+
+    TEST(hashString, testKnownBLAKE3Hashes2) {
+        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:c19012cc2aaf0dc3d8e5c45a1b79114d2df42abb2a410bf54be09e891af06ff8");
+    }
+
     TEST(hashString, testKnownMD5Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s1 = "";