vendor: move hash to vendor (#27262)

* vendor: move hash to vendor

* group hashes into one single static lib

CMakeLists.txt

@@ -227,6 +227,7 @@ add_subdirectory(src)
 if (LLAMA_BUILD_COMMON)
     add_subdirectory(common)
     add_subdirectory(vendor/cpp-httplib)
+    add_subdirectory(vendor/hash)
 endif()
 
 if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)

examples/gguf-hash/CMakeLists.txt

@@ -2,21 +2,5 @@ set(TARGET llama-gguf-hash)
 add_executable(${TARGET} gguf-hash.cpp)
 install(TARGETS ${TARGET} RUNTIME)
 
-# clibs dependencies
+target_link_libraries(${TARGET} PRIVATE vendor-hash ggml ${CMAKE_THREAD_LIBS_INIT})
-include_directories(deps/)
-
-add_library(xxhash OBJECT deps/xxhash/xxhash.c deps/xxhash/xxhash.h)
-target_link_libraries(${TARGET} PRIVATE xxhash)
-
-add_library(sha1 OBJECT deps/sha1/sha1.c deps/sha1/sha1.h)
-target_link_libraries(${TARGET} PRIVATE sha1)
-if (NOT MSVC)
-    # disable warnings in 3rd party code
-    target_compile_options(sha1 PRIVATE -w)
-endif()
-
-add_library(sha256 OBJECT deps/sha256/sha256.c deps/sha256/sha256.h)
-target_link_libraries(${TARGET} PRIVATE sha256)
-
-target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/gguf-hash/deps/rotate-bits/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "rotate-bits",
-  "version": "0.1.1",
-  "repo": "jb55/rotate-bits.h",
-  "description": "rotate bits",
-  "keywords": ["rotl", "rotr"],
-  "src": ["rotate-bits.h"],
-  "license": "Public Domain",
-  "development": {
-    "thlorenz/tap.c": "*"
-  }
-}
-

examples/gguf-hash/deps/sha1/package.json

@@ -1,9 +0,0 @@
-{
-  "name": "sha1",
-  "version": "0.0.1",
-  "repo": "clibs/sha1",
-  "description": "sha1 hash algorithm",
-  "keywords": ["sha1", "hash"],
-  "license": "public domain",
-  "src": ["sha1.c", "sha1.h"]
-}

examples/gguf-hash/deps/sha256/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "sha256",
-  "version": "0.0.2",
-  "repo": "jb55/sha256.c",
-  "description": "sha256 in c",
-  "keywords": ["sha256", "sha2"],
-  "src": ["sha256.c", "sha256.h"],
-  "dependencies": {
-    "jb55/rotate-bits.h": "0.1.1"
-  },
-  "development": {
-    "thlorenz/tap.c": "*"
-  }
-}
-

examples/gguf-hash/deps/xxhash/clib.json

@@ -1,12 +0,0 @@
-{
-  "name": "xxhash",
-  "version": "0.8.2",
-  "repo": "Cyan4973/xxhash",
-  "description": "Extremely fast non-cryptographic hash algorithm",
-  "keywords": ["xxhash", "hashing"],
-  "license": "BSD-2-Clause",
-  "src": [
-    "xxhash.c",
-    "xxhash.h"
-  ]
-}

scripts/sync_vendor.py

@@ -7,6 +7,12 @@ import subprocess
 
 HTTPLIB_VERSION = "refs/tags/v0.53.1"
 
+# used by examples/gguf-hash, these repos have no release tag, so we pin a commit
+XXHASH_COMMIT      = "9f465f1ea932d6ad9a26cd77496311ffa544cd68"
+SHA1_COMMIT        = "e1e2536fcf6a8f9703be8c85d58724b408552287"
+SHA256_COMMIT      = "5e637272c13f200872d55ff579f7e2ab6c3f252f"
+ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1"
+
 vendor = {
     "https://github.com/nlohmann/json/releases/latest/download/json.hpp":     "vendor/nlohmann/json.hpp",
     "https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp",
@@ -22,12 +28,57 @@ vendor = {
     f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE":   "vendor/cpp-httplib/LICENSE",
 
     "https://raw.githubusercontent.com/sheredom/subprocess.h/9ce0d701b6fb10f8f8c4445edd31e7c60a1237e3/subprocess.h": "vendor/sheredom/subprocess.h",
+
+    f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c":      "vendor/hash/xxhash/xxhash.c",
+    f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h":      "vendor/hash/xxhash/xxhash.h",
+    f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE":       "vendor/hash/xxhash/LICENSE",
+
+    # clibs/sha1 ships no license file, the source header says public domain
+    f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/hash/sha1/sha1.c",
+    f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/hash/sha1/sha1.h",
+
+    f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/hash/sha256/sha256.c",
+    f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/hash/sha256/sha256.h",
+    f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE":  "vendor/hash/sha256/LICENSE",
+
+    f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/hash/rotate-bits/rotate-bits.h",
+    f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md":   "vendor/hash/rotate-bits/LICENSE.md",
+}
+
+# local changes kept on top of the upstream sources
+patches = {
+    "vendor/hash/xxhash/xxhash.h": [(
+        '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n',
+        '/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n'
+        '   before allowing the windows compiler to use the C11 form.\n'
+        '   Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n'
+        '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n'
+        '    && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'
+    )],
+
+    # silence a maybe-uninitialized warning
+    "vendor/hash/sha256/sha256.c": [(
+        "  uint32_t W[16];\n",
+        "  uint32_t W[16] = {0};\n"
+    )],
 }
 
 for url, filename in vendor.items():
     print(f"downloading {url} to {filename}") # noqa: NP100
     urllib.request.urlretrieve(url, filename)
 
+for filename, replacements in patches.items():
+    print(f"patching {filename}") # noqa: NP100
+    with open(filename, "r", encoding="utf-8", newline="") as f:
+        content = f.read()
+    for old, new in replacements:
+        if content.count(old) != 1:
+            print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100
+            sys.exit(1)
+        content = content.replace(old, new)
+    with open(filename, "w", encoding="utf-8", newline="") as f:
+        f.write(content)
+
 print("Splitting httplib.h...") # noqa: NP100
 try:
     subprocess.check_call([

vendor/hash/CMakeLists.txt

@@ -0,0 +1,21 @@
+set(TARGET vendor-hash)
+
+llama_add_compile_flags()
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+add_library(${TARGET} STATIC
+    xxhash/xxhash.c
+    sha1/sha1.c
+    sha256/sha256.c
+)
+
+# disable warnings in 3rd party code
+if (CMAKE_C_COMPILER_ID STREQUAL "MSVC")
+    target_compile_options(${TARGET} PRIVATE /w)
+else()
+    target_compile_options(${TARGET} PRIVATE -w)
+endif()
+
+# sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too
+target_include_directories(${TARGET} PUBLIC .)

vendor/hash/rotate-bits/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 William Casarin <jb55@jb55.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/hash/sha256/LICENSE

@@ -0,0 +1 @@
+2010-06-11 : Igor Pavlov : Public domain

vendor/hash/xxhash/LICENSE

@@ -0,0 +1,26 @@
+xxHash Library
+Copyright (c) 2012-2021 Yann Collet
+All rights reserved.
+
+BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.