Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
adc2aa3ecb | ||
|
|
fe57bc93a0 | ||
|
|
d9ca3278bf | ||
|
|
cb48bf2240 | ||
|
|
18691e0090 | ||
|
|
2c6c517ba5 | ||
|
|
2874dcb107 | ||
|
|
dd851fd0c9 | ||
|
|
f24c3582c2 | ||
|
|
71d4cbe195 | ||
|
|
2fc9f9c351 | ||
|
|
cce0ac1f09 | ||
|
|
d7b3510499 | ||
|
|
47b892271c | ||
|
|
36514dba27 | ||
|
|
e5fc474b87 | ||
|
|
ceab1a91f8 | ||
|
|
80e662a2a1 | ||
|
|
2512290f94 | ||
|
|
62d442e7bf | ||
|
|
f664b0f20e | ||
|
|
f5c0d8f28a | ||
|
|
bfbaf15b2a | ||
|
|
149a8cb0ae | ||
|
|
e085c347d2 | ||
|
|
6c480f7e70 | ||
|
|
11044b35e3 | ||
|
|
54a7f534b6 | ||
|
|
7ac2014a80 | ||
|
|
53f9077719 | ||
|
|
878d1338f4 | ||
|
|
4c4486f8e9 | ||
|
|
5e429d4a9c | ||
|
|
b7c131219c | ||
|
|
5e99ac1982 | ||
|
|
0a0f3ab6d4 | ||
|
|
ad27d49218 | ||
|
|
9b555f1aad | ||
|
|
9857a45f0c | ||
|
|
bf2c1a6992 | ||
|
|
2feb6aeb27 | ||
|
|
8d80f0673f | ||
|
|
b9df854de8 | ||
|
|
958257cd60 | ||
|
|
17165fe2a7 | ||
|
|
481be8b284 | ||
|
|
3df719e1e8 | ||
|
|
ff2a96d2d4 | ||
|
|
fb6a8a0761 | ||
|
|
c72cb424e1 |
No files matched your search
@@ -19,16 +19,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
@@ -44,7 +40,7 @@ WORKDIR /app
|
||||
# -- Copy project files --
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
# -- Set CANN environment variables (required for compilation) --
|
||||
# Using ENV instead of `source` allows environment variables to persist across the entire image layer
|
||||
|
||||
+8
-10
@@ -1,4 +1,4 @@
|
||||
ARG UBUNTU_VERSION=26.04
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
ARG APP_REVISION=N/A
|
||||
@@ -9,16 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -26,13 +22,15 @@ FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential git cmake libssl-dev
|
||||
apt-get install -y gcc-14 g++-14 build-essential git cmake libssl-dev
|
||||
|
||||
ENV CC=gcc-14 CXX=g++-14
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
|
||||
|
||||
+4
-11
@@ -17,16 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
@@ -44,7 +40,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
|
||||
@@ -88,9 +84,6 @@ RUN apt-get update \
|
||||
&& find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
|
||||
&& find /var/cache -type f -delete
|
||||
|
||||
# Some hosts (e.g. GKE) only mount the driver here, and CUDA 12.8.1-12.9.1 base images don't set these paths
|
||||
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/nvidia/lib:${LD_LIBRARY_PATH}
|
||||
|
||||
COPY --from=build /app/lib/ /app
|
||||
|
||||
### Full
|
||||
|
||||
+17
-23
@@ -1,4 +1,4 @@
|
||||
ARG ONEAPI_VERSION=2026.2.0-devel-ubuntu24.04
|
||||
ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
ARG APP_REVISION=N/A
|
||||
@@ -11,41 +11,37 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
|
||||
RUN echo "Download oneDNN installation package" && \
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh && \
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh && \
|
||||
echo "Install oneDNN components" && \
|
||||
bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
|
||||
ARG GGML_SYCL_F16=ON
|
||||
ARG LEVEL_ZERO_VERSION=1.34.0
|
||||
ARG LEVEL_ZERO_VERSION=1.28.2
|
||||
ARG LEVEL_ZERO_UBUNTU_VERSION=u24.04
|
||||
RUN apt-get update && \
|
||||
apt-get install -y git libssl-dev wget ca-certificates && \
|
||||
cd /tmp && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb && \
|
||||
apt-get -o Dpkg::Options::="--force-overwrite" install -y ./libze1.deb ./libze-dev.deb && \
|
||||
rm -f /tmp/libze1.deb /tmp/libze-dev.deb
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb && \
|
||||
apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb && \
|
||||
rm -f /tmp/level-zero.deb /tmp/level-zero-devel.deb
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
echo "GGML_SYCL_F16 is set" \
|
||||
@@ -68,7 +64,7 @@ RUN mkdir -p /app/full \
|
||||
&& cp requirements.txt /app/full \
|
||||
&& cp .devops/tools.sh /app/full/tools.sh
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
|
||||
FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
|
||||
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
@@ -83,8 +79,6 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
|
||||
org.opencontainers.image.url=$IMAGE_URL \
|
||||
org.opencontainers.image.source=$IMAGE_SOURCE
|
||||
|
||||
COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
|
||||
|
||||
#Following versions are for multiple GPUs, since 26.x has known issue:
|
||||
# https://github.com/ggml-org/llama.cpp/issues/21747,
|
||||
# https://github.com/intel/compute-runtime/issues/921.
|
||||
@@ -95,11 +89,11 @@ COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
|
||||
#ARG IGDGMM_VERSION=22.8.2
|
||||
|
||||
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
ARG IGC_VERSION=v2.34.4
|
||||
ARG IGC_VERSION_FULL=2_2.34.4+21428
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
|
||||
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
|
||||
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
|
||||
|
||||
@@ -15,16 +15,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
@@ -49,7 +45,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
|
||||
|
||||
+41
-138
@@ -22,19 +22,12 @@
|
||||
spirv-headers,
|
||||
nodejs,
|
||||
importNpmLock,
|
||||
syclToolkit ? null,
|
||||
level-zero,
|
||||
ocl-icd,
|
||||
intel-graphics-compiler,
|
||||
intel-compute-runtime,
|
||||
makeBinaryWrapper,
|
||||
useBlas ?
|
||||
builtins.all (x: !x) [
|
||||
useCuda
|
||||
useMetalKit
|
||||
useRocm
|
||||
useVulkan
|
||||
useSycl
|
||||
]
|
||||
&& blas.meta.available,
|
||||
useCuda ? config.cudaSupport,
|
||||
@@ -45,28 +38,11 @@
|
||||
rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,
|
||||
useVulkan ? false,
|
||||
useRpc ? false,
|
||||
useSycl ? false,
|
||||
# Rebuild, not a runtime switch. cmake default is off.
|
||||
syclF16 ? false,
|
||||
# null keeps JIT. AOT makes icpx run ocloc, which DPC++ does not ship.
|
||||
syclDeviceArch ? null,
|
||||
syclMaxParallelLinkJobs ? null,
|
||||
llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
|
||||
|
||||
# CUDA needs its backend stdenv or libstdc++ mismatches downstream.
|
||||
# SYCL needs icpx which is provided by the oneAPI stdenv
|
||||
effectiveStdenv ?
|
||||
if useCuda && useSycl then
|
||||
throw "llama-cpp: useCuda and useSycl are mutually exclusive"
|
||||
else if useCuda then
|
||||
cudaPackages.backendStdenv
|
||||
else if useSycl then
|
||||
if syclToolkit == null then
|
||||
throw "llama-cpp: useSycl requires syclToolkit, consider setting it to intel-oneapi-toolkit"
|
||||
else
|
||||
syclToolkit.stdenv
|
||||
else
|
||||
stdenv,
|
||||
# It's necessary to consistently use backendStdenv when building with CUDA support,
|
||||
# otherwise we get libstdc++ errors downstream.
|
||||
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
|
||||
enableStatic ? effectiveStdenv.hostPlatform.isStatic,
|
||||
precompileMetalShaders ? false,
|
||||
useWebUi ? true,
|
||||
@@ -89,8 +65,7 @@ let
|
||||
++ lib.optionals useMetalKit [ "MetalKit" ]
|
||||
++ lib.optionals useMpi [ "MPI" ]
|
||||
++ lib.optionals useRocm [ "ROCm" ]
|
||||
++ lib.optionals useVulkan [ "Vulkan" ]
|
||||
++ lib.optionals useSycl [ "SYCL" ];
|
||||
++ lib.optionals useVulkan [ "Vulkan" ];
|
||||
|
||||
pnameSuffix =
|
||||
strings.optionalString (suffices != [ ])
|
||||
@@ -133,26 +108,6 @@ let
|
||||
shaderc
|
||||
spirv-headers
|
||||
];
|
||||
|
||||
syclAot = useSycl && syclDeviceArch != null && syclDeviceArch != "";
|
||||
|
||||
# source ocloc binary from intel-compute-runtime
|
||||
oclocAot = runCommand "ocloc-aot" { nativeBuildInputs = [ makeBinaryWrapper ]; } ''
|
||||
mkdir -p "$out/bin"
|
||||
shopt -s nullglob
|
||||
exact=(${lib.getBin intel-compute-runtime}/bin/ocloc)
|
||||
vers=(${lib.getBin intel-compute-runtime}/bin/ocloc-*)
|
||||
if [ ''${#exact[@]} -eq 1 ]; then
|
||||
src=''${exact[0]}
|
||||
elif [ ''${#vers[@]} -eq 1 ]; then
|
||||
src=''${vers[0]}
|
||||
else
|
||||
echo "expected one ocloc binary from intel-compute-runtime" >&2
|
||||
exit 1
|
||||
fi
|
||||
makeWrapper "$src" "$out/bin/ocloc" \
|
||||
--prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ intel-graphics-compiler ]}
|
||||
'';
|
||||
in
|
||||
|
||||
effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
@@ -178,13 +133,11 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
src = lib.cleanSource ../../.;
|
||||
};
|
||||
|
||||
# Builds Llama Chat locally, taking care not to require updating any sha256 hash.
|
||||
# Builds the webui locally, taking care not to require updating any sha256 hash.
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
# the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole
|
||||
src = lib.cleanSource ../../ui;
|
||||
sourceRoot = "source/apps/chat";
|
||||
src = lib.cleanSource ../../tools/ui;
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -193,7 +146,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# no sha256 required when using buildNodeModules
|
||||
npmDeps = importNpmLock.buildNodeModules {
|
||||
npmRoot = ../../ui/apps/chat;
|
||||
npmRoot = ../../tools/ui;
|
||||
inherit nodejs;
|
||||
};
|
||||
|
||||
@@ -203,8 +156,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
};
|
||||
|
||||
postPatch = lib.optionalString useWebUi ''
|
||||
cp -r ${finalAttrs.webui} ui/apps/chat/dist
|
||||
chmod -R u+w ui/apps/chat/dist
|
||||
cp -r ${finalAttrs.webui} tools/ui/dist
|
||||
chmod -R u+w tools/ui/dist
|
||||
'';
|
||||
|
||||
# With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
|
||||
@@ -215,13 +168,6 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
# see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
|
||||
__noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
|
||||
# Do not fortify under SYCL as it breaks compiled memcpy invocations
|
||||
# _FORTIFY_SOURCE rewrites memcpy -> __memcpy_chk
|
||||
hardeningDisable = optionals useSycl [
|
||||
"fortify"
|
||||
"fortify3"
|
||||
];
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
cmake
|
||||
@@ -235,12 +181,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
autoAddDriverRunpath
|
||||
]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]
|
||||
++ optionals useSycl [
|
||||
autoAddDriverRunpath
|
||||
makeBinaryWrapper
|
||||
]
|
||||
++ optionals syclAot [ oclocAot ];
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
|
||||
buildInputs =
|
||||
optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
|
||||
@@ -249,53 +190,39 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
++ optionals useRocm rocmBuildInputs
|
||||
++ optionals useBlas [ blas ]
|
||||
++ optionals useVulkan vulkanBuildInputs
|
||||
++ optionals useSycl [
|
||||
level-zero
|
||||
ocl-icd
|
||||
intel-graphics-compiler
|
||||
]
|
||||
++ [ openssl ];
|
||||
|
||||
cmakeFlags = [
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
cmakeFlags =
|
||||
[
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
)
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
]
|
||||
++ optionals useSycl [
|
||||
(cmakeBool "GGML_SYCL" true)
|
||||
(cmakeBool "GGML_SYCL_F16" syclF16)
|
||||
]
|
||||
++ optionals syclAot [
|
||||
(cmakeFeature "GGML_SYCL_DEVICE_ARCH" syclDeviceArch)
|
||||
]
|
||||
++ optionals (syclAot && syclMaxParallelLinkJobs != null) [
|
||||
(cmakeFeature "GGML_SYCL_MAX_PARALLEL_LINK_JOBS" (toString syclMaxParallelLinkJobs))
|
||||
];
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
];
|
||||
|
||||
# Environment variables needed for ROCm
|
||||
env = optionalAttrs useRocm {
|
||||
@@ -310,35 +237,11 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
cp $src/include/llama.h $out/include/
|
||||
'';
|
||||
|
||||
# Fix SYCL binary calling missing important libraries
|
||||
# Also use Level Zero + sysman INSTEAD of OpenCL
|
||||
# due to OpenCL reporting free memory as total memory,
|
||||
# breaking `-fit` functionality
|
||||
postFixup = lib.optionalString useSycl ''
|
||||
driverLibs=${
|
||||
lib.makeLibraryPath [
|
||||
intel-graphics-compiler
|
||||
ocl-icd
|
||||
]
|
||||
}
|
||||
for b in "$out"/bin/*; do
|
||||
[ -f "$b" ] && [ -x "$b" ] || continue
|
||||
patchelf --print-interpreter "$b" >/dev/null 2>&1 || continue
|
||||
patchelf --add-needed libOpenCL.so.1 --add-needed libigc.so.2 \
|
||||
--add-rpath "$driverLibs" "$b"
|
||||
wrapProgram "$b" \
|
||||
--set-default ONEAPI_DEVICE_SELECTOR level_zero:gpu \
|
||||
--set-default ZES_ENABLE_SYSMAN 1
|
||||
done
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# Configurations we don't want even the CI to evaluate. Results in the
|
||||
# "unsupported platform" messages. This is mostly a no-op, because
|
||||
# cudaPackages would've refused to evaluate anyway.
|
||||
badPlatforms =
|
||||
optionals useCuda lib.platforms.darwin
|
||||
++ optionals useSycl (lib.subtractLists [ "x86_64-linux" ] lib.platforms.all);
|
||||
badPlatforms = optionals useCuda lib.platforms.darwin;
|
||||
|
||||
# Configurations that are known to result in build failures. Can be
|
||||
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||
|
||||
@@ -22,6 +22,10 @@ let
|
||||
gitpython
|
||||
tabulate
|
||||
|
||||
# for examples/pydantic-models-to-grammar-examples.py
|
||||
docstring-parser
|
||||
pydantic
|
||||
|
||||
];
|
||||
|
||||
llama-python-test-deps = with python3Packages; [
|
||||
|
||||
@@ -3,11 +3,11 @@ ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
ARG IGC_VERSION=v2.41.5
|
||||
ARG IGC_VERSION_FULL=2_2.41.5+22716
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
ARG NPU_DRIVER_VERSION=v1.38.0
|
||||
@@ -28,16 +28,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
@@ -87,7 +83,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
# Build Stage
|
||||
RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
|
||||
|
||||
@@ -17,16 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
@@ -56,7 +52,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
||||
cmake -S . -B build \
|
||||
|
||||
+15
-13
@@ -9,16 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -35,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
@@ -86,14 +82,20 @@ COPY --from=build /app/full /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}"
|
||||
|
||||
# Flag for compatibility with pip
|
||||
ARG UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y \
|
||||
build-essential \
|
||||
curl \
|
||||
git \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
&& pip install --break-system-packages --upgrade setuptools \
|
||||
&& pip install --break-system-packages -r requirements.txt \
|
||||
ca-certificates \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
|
||||
&& uv python install 3.13 \
|
||||
&& uv venv --python 3.13 /root/.venv \
|
||||
&& uv pip install --python /root/.venv/bin/python -r requirements.txt \
|
||||
&& apt autoremove -y \
|
||||
&& apt clean -y \
|
||||
&& rm -rf /tmp/* /var/tmp/* \
|
||||
|
||||
@@ -9,16 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -32,7 +28,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
build*/
|
||||
|
||||
ui/apps/chat/node_modules/
|
||||
tools/ui/node_modules/
|
||||
|
||||
models/*
|
||||
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ insert_final_newline = unset
|
||||
[vendor/**.patch]
|
||||
trim_trailing_whitespace = unset
|
||||
|
||||
[ui/**]
|
||||
[tools/ui/**]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
end_of_line = unset
|
||||
|
||||
@@ -14,9 +14,7 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" && "${{ env.BRANCH_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "name=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
||||
elif [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
@@ -25,9 +25,7 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
@@ -56,9 +54,7 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
@@ -89,9 +85,7 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
@@ -106,60 +100,56 @@ runs:
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.5 for x64
|
||||
if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'x64' }}
|
||||
- name: Install Cuda Toolkit 13.4 for x64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvrtc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-x86_64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvtx-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_profiler_api-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\visual_studio_integration-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.5 for ARM64
|
||||
if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'arm64' }}
|
||||
- name: Install Cuda Toolkit 13.4 for ARM64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-arm64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
+2
-1
@@ -48,6 +48,7 @@ examples:
|
||||
- all-globs-to-all-files:
|
||||
- '!tools/server/**'
|
||||
- '!tools/mtmd/**'
|
||||
- '!tools/ui/**'
|
||||
testing:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -71,7 +72,7 @@ android:
|
||||
server/ui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- ui/**
|
||||
- tools/ui/**
|
||||
server:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
|
||||
@@ -13,8 +13,6 @@ on:
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.py',
|
||||
'**/requirements*.txt',
|
||||
]
|
||||
|
||||
pull_request:
|
||||
@@ -26,9 +24,7 @@ on:
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.py',
|
||||
'**/requirements*.txt',
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
@@ -73,7 +69,7 @@ jobs:
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
pip install jinja2==3.1.6
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
@@ -111,67 +107,3 @@ jobs:
|
||||
source .venv/bin/activate
|
||||
cd build
|
||||
ctest -L behavior -LE 'model|backend' --verbose --timeout 900
|
||||
|
||||
windows:
|
||||
runs-on: windows-2025
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
key: behavior-check-windows-2025
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
shell: bash
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
run: |
|
||||
call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
|
||||
cmake -B build -G "Ninja Multi-Config" ^
|
||||
-DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
|
||||
-DCMAKE_BUILD_TYPE=Release ^
|
||||
-DGGML_NATIVE=OFF ^
|
||||
-DGGML_BACKEND_DL=OFF
|
||||
set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
|
||||
cmake --build build --config Release -j %NINJA_JOBS%
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
shell: bash
|
||||
run: |
|
||||
source .venv/Scripts/activate
|
||||
cd build
|
||||
ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: behavior-check-windows-2025
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
@@ -31,8 +31,8 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-llguidance:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
ubuntu-24-llguidance:
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -33,7 +33,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
android-ndk-snapdragon:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8'
|
||||
defaults:
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
path: pkg-snapdragon/llama.cpp
|
||||
|
||||
linux-iot-snapdragon:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8'
|
||||
defaults:
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
test-snapdragon-qdc:
|
||||
name: Test on QDC Device (${{ matrix.device }})
|
||||
needs: [android-ndk-snapdragon, linux-iot-snapdragon]
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 180
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
||||
@@ -37,7 +37,7 @@ env:
|
||||
|
||||
jobs:
|
||||
default:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -56,7 +56,6 @@ jobs:
|
||||
uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
|
||||
with:
|
||||
log-accepted-android-sdk-licenses: false
|
||||
packages: platform-tools cmake;3.31.6
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
@@ -64,7 +63,7 @@ jobs:
|
||||
./gradlew build --no-daemon
|
||||
|
||||
ndk:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
|
||||
defaults:
|
||||
@@ -102,7 +101,7 @@ jobs:
|
||||
path: pkg-adb/llama.cpp
|
||||
|
||||
arm64:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
env:
|
||||
NDK_VERSION: "29.0.14206865"
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
name: CI (apple - daily)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Xcode
|
||||
uses: ggml-org/setup-xcode@v1
|
||||
with:
|
||||
xcode-version: latest-stable
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
|
||||
- name: Upload xcframework artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
retention-days: 1
|
||||
|
||||
- name: Build Xcode project
|
||||
run: |
|
||||
xcodebuild -downloadPlatform iOS
|
||||
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
|
||||
macos-latest-tvos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-visionos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-swift:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download xcframework artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
|
||||
- name: Build llama.cpp with CMake
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
@@ -107,3 +107,196 @@ jobs:
|
||||
# ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
|
||||
ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
|
||||
|
||||
macos-latest-x64:
|
||||
runs-on: macos-15-intel
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: apple-x64
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
# Metal is disabled due to intermittent failures with Github runners not having a GPU:
|
||||
# https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_RPATH="@loader_path" \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_BUILD_BORINGSSL=ON \
|
||||
-DGGML_METAL=OFF \
|
||||
-DGGML_RPC=ON \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
|
||||
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: apple-x64
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Xcode
|
||||
uses: ggml-org/setup-xcode@v1
|
||||
with:
|
||||
xcode-version: latest-stable
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
|
||||
- name: Upload xcframework artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
retention-days: 1
|
||||
|
||||
- name: Build Xcode project
|
||||
run: |
|
||||
xcodebuild -downloadPlatform iOS
|
||||
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
|
||||
macos-latest-tvos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-visionos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-swift:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download xcframework artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
|
||||
- name: Build llama.cpp with CMake
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
@@ -40,8 +40,8 @@ jobs:
|
||||
# path: ./spacemit_toolchain
|
||||
# version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
|
||||
|
||||
ubuntu-26-openvino-cache:
|
||||
runs-on: ubuntu-26.04
|
||||
ubuntu-24-openvino-cache:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Build relocatable cmake package
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
runs-on: [self-hosted, Linux, CPU]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
PREFIX="$(pwd)"/inst
|
||||
cmake -S . -B build \
|
||||
-DCMAKE_PREFIX_PATH="$PREFIX" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_IS_DEV=OFF \
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
cmake --install build --prefix "$PREFIX" --config Release
|
||||
|
||||
export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
|
||||
build_commit=$(git rev-parse --short HEAD | xargs)
|
||||
build_number=$(git rev-list --count HEAD | xargs)
|
||||
|
||||
major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
build_version="$major.$minor.$patch"
|
||||
|
||||
checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
|
||||
"set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
|
||||
"set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
|
||||
|
||||
for check in "${checks[@]}"; do
|
||||
if ! grep -qE "$check" "$LLAMA_CONFIG"; then
|
||||
echo "Checking llama-config.cmake version... \"$check\" failed!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Checking llama-config.cmake version... success."
|
||||
|
||||
cd examples/simple-cmake-pkg
|
||||
cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
|
||||
cmake --build build -j $(nproc)
|
||||
@@ -7,6 +7,7 @@ on:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-cpu.yml',
|
||||
'.github/workflows/build-cmake-pkg.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
@@ -19,6 +20,7 @@ on:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-cpu.yml',
|
||||
'.github/workflows/build-cmake-pkg.yml',
|
||||
'ggml/src/ggml-rpc/**',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
@@ -45,14 +47,17 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
build-cmake-pkg:
|
||||
uses: ./.github/workflows/build-cmake-pkg.yml
|
||||
|
||||
ubuntu:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- build: 'x64'
|
||||
os: ubuntu-26.04
|
||||
os: ubuntu-22.04
|
||||
- build: 'arm64'
|
||||
os: ubuntu-26.04-arm
|
||||
os: ubuntu-24.04-arm
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@@ -76,6 +81,13 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
@@ -187,11 +199,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
@@ -235,79 +242,3 @@ jobs:
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
build-cmake-pkg:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: build-cmake-pkg
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
PREFIX="$(pwd)"/inst
|
||||
cmake -S . -B build \
|
||||
-DCMAKE_PREFIX_PATH="$PREFIX" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_IS_DEV=OFF \
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
cmake --install build --prefix "$PREFIX" --config Release
|
||||
|
||||
export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
|
||||
build_commit=$(git rev-parse --short HEAD | xargs)
|
||||
build_number=$(git rev-list --count HEAD | xargs)
|
||||
|
||||
major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
build_version="$major.$minor.$patch"
|
||||
|
||||
checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
|
||||
"set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
|
||||
"set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
|
||||
|
||||
for check in "${checks[@]}"; do
|
||||
if ! grep -qE "$check" "$LLAMA_CONFIG"; then
|
||||
echo "Checking llama-config.cmake version... \"$check\" failed!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Checking llama-config.cmake version... success."
|
||||
|
||||
cd examples/simple-cmake-pkg
|
||||
cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: build-cmake-pkg
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
@@ -163,7 +163,7 @@ jobs:
|
||||
# cmake --build build --config Release -j $(nproc)
|
||||
|
||||
debian-13-loongarch64-cpu-cross:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
|
||||
|
||||
steps:
|
||||
@@ -218,7 +218,7 @@ jobs:
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
debian-13-loongarch64-vulkan-cross:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
|
||||
|
||||
steps:
|
||||
@@ -276,8 +276,8 @@ jobs:
|
||||
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
ubuntu-26-riscv64-cpu-spacemit-ime-cross:
|
||||
runs-on: ubuntu-26.04
|
||||
ubuntu-24-riscv64-cpu-spacemit-ime-cross:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
# Make sure this is in sync with build-cache.yml
|
||||
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
|
||||
hip:
|
||||
runs-on: ubuntu-22.04
|
||||
container: rocm/dev-ubuntu-22.04:6.3.4
|
||||
container: rocm/dev-ubuntu-22.04:6.1.2
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -40,12 +40,12 @@ jobs:
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.5'
|
||||
- cuda: '13.4'
|
||||
arch: x64
|
||||
defines: ''
|
||||
- cuda: '13.5'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.4'
|
||||
arch: arm64
|
||||
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -43,9 +43,9 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-s390x:
|
||||
name: ubuntu-26-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
ubuntu-24-s390x:
|
||||
name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -66,6 +66,12 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
@@ -100,7 +106,7 @@ jobs:
|
||||
run: |
|
||||
cd build
|
||||
echo "Fetch llama2c big-endian model"
|
||||
wget https://huggingface.co/ggml-org-dev/stories260K/resolve/main/stories260K-be.gguf
|
||||
wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
|
||||
./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
||||
|
||||
- name: Test quantize (s390x)
|
||||
@@ -110,19 +116,30 @@ jobs:
|
||||
|
||||
ubuntu-26-zdnn-s390x:
|
||||
name: ubuntu-26-zdnn-s390x
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
container: ubuntu:26.04 # required to get GCC 15.1 and binutils 2.44
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates \
|
||||
libssl-dev libzdnn-dev
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
- name: Toolchain workaround (GCC 15)
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential libssl-dev libzdnn-dev
|
||||
apt-get install -y gcc-15 g++-15
|
||||
echo "CC=gcc-15" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-15" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build with zDNN Backend
|
||||
id: cmake_build
|
||||
@@ -137,8 +154,8 @@ jobs:
|
||||
-DCMAKE_CXX_FLAGS="-march=arch15"
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
ubuntu-26-ppc64le:
|
||||
runs-on: ubuntu-26.04-ppc64le
|
||||
ubuntu-24-ppc64le:
|
||||
runs-on: ubuntu-24.04-ppc64le
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -154,6 +171,12 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
|
||||
@@ -61,11 +61,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -37,9 +37,64 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXCLUDE: "test-llama-archs|test-llama-context|test-fp8-kv"
|
||||
CTEST_EXCLUDE: "test-llama-archs|test-llama-context"
|
||||
|
||||
jobs:
|
||||
ubuntu-24-openvino:
|
||||
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip
|
||||
sudo apt-get install -y ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
|
||||
- name: Setup OpenVINO Toolkit
|
||||
uses: ./.github/actions/linux-setup-openvino
|
||||
with:
|
||||
path: ./openvino_toolkit
|
||||
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
||||
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
||||
|
||||
- name: Install OpenVINO dependencies
|
||||
run: |
|
||||
cd ./openvino_toolkit
|
||||
chmod +x ./install_dependencies/install_openvino_dependencies.sh
|
||||
echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
source ./openvino_toolkit/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_OPENVINO=ON
|
||||
time cmake --build build/ReleaseOV --config Release --parallel
|
||||
|
||||
- name: Test (CPU)
|
||||
id: cmake_test_cpu
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
|
||||
- name: Test (GPU)
|
||||
id: cmake_test_gpu
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
|
||||
openvino-windows-2022:
|
||||
runs-on: windows-2022
|
||||
|
||||
|
||||
@@ -43,12 +43,13 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# thread and address doesn't run properly on some self hosted machines, so run it on Github instead
|
||||
- sanitizer: ADDRESS
|
||||
machine: ubuntu-26.04
|
||||
machine: ubuntu-24.04
|
||||
- sanitizer: THREAD
|
||||
machine: ubuntu-26.04
|
||||
machine: ubuntu-24.04
|
||||
- sanitizer: UNDEFINED
|
||||
machine: ubuntu-26.04
|
||||
machine: [self-hosted, X64, Linux]
|
||||
|
||||
runs-on: ${{ matrix.machine }}
|
||||
|
||||
@@ -57,24 +58,31 @@ jobs:
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# if: ${{ matrix.sanitizer != 'UNDEFINED' }}
|
||||
# with:
|
||||
# key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
|
||||
# variant: ccache
|
||||
# evict-old-files: 1d
|
||||
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: ctest-sanitizer-${{ matrix.sanitizer }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
# with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
|
||||
- name: Build (undefined)
|
||||
id: cmake_build_undefined
|
||||
if: ${{ matrix.sanitizer == 'UNDEFINED' }}
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Debug \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
||||
-DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
|
||||
|
||||
cmake --build build --config Debug -j $(nproc)
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
if: ${{ matrix.sanitizer == 'ADDRESS' || matrix.sanitizer == 'UNDEFINED' }}
|
||||
if: ${{ matrix.sanitizer == 'ADDRESS' }}
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
|
||||
@@ -95,18 +103,6 @@ jobs:
|
||||
|
||||
cmake --build build --config RelWithDebInfo -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: ctest-sanitizer-${{ matrix.sanitizer }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
|
||||
@@ -38,7 +38,7 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-sycl:
|
||||
ubuntu-24-sycl:
|
||||
strategy:
|
||||
matrix:
|
||||
build: [fp32, fp16]
|
||||
@@ -48,12 +48,12 @@ jobs:
|
||||
- build: fp16
|
||||
fp16: ON
|
||||
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
LEVEL_ZERO_VERSION: "1.34.0"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
continue-on-error: true
|
||||
@@ -68,12 +68,12 @@ jobs:
|
||||
run: |
|
||||
cd /tmp
|
||||
echo "Download oneAPI Deep Learning Essentials installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
echo "Install oneAPI Deep Learning Essentials components"
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
echo "Download oneDNN installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh
|
||||
echo "Install oneDNN components"
|
||||
sudo bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: sycl-ubuntu-26-${{ matrix.build }}
|
||||
key: sycl-ubuntu-24-${{ matrix.build }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: sycl-ubuntu-26-${{ matrix.build }}
|
||||
key: sycl-ubuntu-24-${{ matrix.build }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
@@ -150,13 +150,13 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
|
||||
WINDOWS_DNNL: intel.oneapi.win.dnnl
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
|
||||
@@ -31,8 +31,8 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
ubuntu-26-virtgpu:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
ubuntu-24-virtgpu:
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -42,7 +42,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ubuntu-arm64:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -53,7 +53,9 @@ jobs:
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
|
||||
sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -67,7 +69,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-arm
|
||||
key: vulkan-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -90,14 +92,14 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-arm
|
||||
key: vulkan-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
ubuntu-llvmpipe:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -135,7 +137,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-llvmpipe
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -154,7 +156,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-llvmpipe
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
@@ -213,11 +215,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -49,7 +49,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ubuntu-webgpu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasi
|
||||
key: webgpu-ubuntu-24.04-arm-wasi
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasi
|
||||
key: webgpu-ubuntu-24.04-arm-wasi
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -50,7 +50,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ubuntu-webgpu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasm
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasm
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -34,7 +34,7 @@ env:
|
||||
|
||||
jobs:
|
||||
format:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key |
|
||||
sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
||||
sudo add-apt-repository -y \
|
||||
"deb http://apt.llvm.org/resolute/ llvm-toolchain-resolute-22 main"
|
||||
"deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang-format-22
|
||||
|
||||
|
||||
@@ -46,55 +46,18 @@ env:
|
||||
|
||||
jobs:
|
||||
cpu-x64-high-perf:
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
# note: this runner has 8 CPUs, but std::thread::hardware_concurrency reports the cores of the
|
||||
# host (https://github.com/ggml-org/llama.cpp/pull/31439#issuecomment-6683386059)
|
||||
env:
|
||||
TEST_ARG_HARDWARE_CONCURRENCY: 8
|
||||
TEST_ARG_PARALLEL_WORKERS: 4
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libssl-dev time unzip wget python3 python3-venv python3-dev
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-cpu-x64-high-perf
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-cpu-x64-high-perf
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
cpu-arm64-high-perf-graviton4:
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
|
||||
@@ -93,72 +93,13 @@ jobs:
|
||||
save: true
|
||||
|
||||
gpu-rocm:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Strix Halo (iGPU, RDNA3.5)
|
||||
- gpu: gfx1151
|
||||
runner: [self-hosted, Linux, strix-halo, devlab-dispatch]
|
||||
# async-execution correctness workaround, gfx1151 only (see Test step)
|
||||
hip_launch_blocking: "1"
|
||||
# Radeon AI PRO R9700 (dGPU, RDNA4)
|
||||
- gpu: gfx1201
|
||||
runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch]
|
||||
hip_launch_blocking: "0"
|
||||
# HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit)
|
||||
disable_hip_graphs: true
|
||||
|
||||
name: gpu-rocm (${{ matrix.gpu }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 120
|
||||
|
||||
env:
|
||||
ROCM_VERSION: "10.0.0"
|
||||
runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Verify GPU device
|
||||
run: |
|
||||
ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; }
|
||||
|
||||
# same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install
|
||||
- name: Setup TheRock with Wheels
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/rocm"
|
||||
"$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}"
|
||||
ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root)
|
||||
{
|
||||
echo "ROCM_PATH=$ROCM_PATH"
|
||||
echo "HIP_PATH=$ROCM_PATH"
|
||||
echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)"
|
||||
echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}"
|
||||
} >> "$GITHUB_ENV"
|
||||
"$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH"
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
# ROCm is installed again on each run, so hash the compiler content, not its mtime
|
||||
- name: Tune ccache for reinstalled ROCm toolchain
|
||||
run: |
|
||||
ccache --set-config=compiler_check=content
|
||||
ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
||||
@@ -166,25 +107,10 @@ jobs:
|
||||
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
||||
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
||||
env:
|
||||
HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }}
|
||||
HIP_LAUNCH_BLOCKING: "1"
|
||||
run: |
|
||||
rocminfo
|
||||
# Exported rather than set in env:, because ggml only checks that the
|
||||
# variable is defined: an empty value would disable graphs on gfx1151 too.
|
||||
if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp"
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-rocm:
|
||||
|
||||
@@ -21,9 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-openvino.yml',
|
||||
'ci/run.sh',
|
||||
'CMakeLists.txt',
|
||||
'ggml/CMakeLists.txt',
|
||||
'tests/CMakeLists.txt',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
|
||||
@@ -16,7 +16,7 @@ cache-mode: none
|
||||
jobs:
|
||||
# The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
|
||||
copilot-setup-steps:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
# Set the permissions to the lowest permissions possible needed for your steps.
|
||||
# Copilot will be given its own token for its operations.
|
||||
|
||||
@@ -16,10 +16,6 @@ on:
|
||||
description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)"
|
||||
type: boolean
|
||||
default: false
|
||||
skip_ref_check:
|
||||
description: "Skip the requirement to run from a tag (allows branch refs)"
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
@@ -42,14 +38,6 @@ jobs:
|
||||
source_tag: ${{ steps.srctag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: Check manual run ref
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_ref_check }}
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then
|
||||
echo "::error::Manual runs of this workflow must be started from a tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -82,7 +70,7 @@ jobs:
|
||||
|
||||
prepare_matrices:
|
||||
name: Prepare Docker matrices
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
build_matrix: ${{ steps.matrices.outputs.build_matrix }}
|
||||
merge_matrix: ${{ steps.matrices.outputs.merge_matrix }}
|
||||
@@ -99,19 +87,19 @@ jobs:
|
||||
# Keep all build targets in one place and derive merge targets from it.
|
||||
cat > build-matrix.json <<'JSON'
|
||||
[
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-s390x", "prebuilt_ui": true },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" }
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }
|
||||
]
|
||||
JSON
|
||||
|
||||
@@ -120,9 +108,6 @@ jobs:
|
||||
mv build-matrix.json.tmp build-matrix.json
|
||||
fi
|
||||
|
||||
jq 'map(select(.tag != "musa"))' build-matrix.json > build-matrix.json.tmp
|
||||
mv build-matrix.json.tmp build-matrix.json
|
||||
|
||||
BUILD_MATRIX="$(jq -c . build-matrix.json)"
|
||||
MERGE_MATRIX="$(jq -c '
|
||||
reduce .[] as $entry ({}; .[$entry.tag] |= (
|
||||
@@ -185,7 +170,7 @@ jobs:
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
@@ -415,7 +400,7 @@ jobs:
|
||||
merge_arch_tags:
|
||||
name: Create shared tags from digests
|
||||
needs: [prepare_matrices, push_to_registry, create_tag]
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
@@ -24,7 +24,7 @@ permissions:
|
||||
jobs:
|
||||
deploy:
|
||||
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -39,7 +39,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
make-release:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@@ -11,8 +11,7 @@ on:
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/llama-graph*',
|
||||
'src/llama-kv-cache*',
|
||||
'src/llama-graph.cpp',
|
||||
'src/llama-model*',
|
||||
'src/models/**'
|
||||
]
|
||||
@@ -25,8 +24,7 @@ on:
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/llama-graph*',
|
||||
'src/llama-kv-cache*',
|
||||
'src/llama-graph.cpp',
|
||||
'src/llama-model*',
|
||||
'src/models/**'
|
||||
]
|
||||
@@ -147,11 +145,6 @@ jobs:
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
- name: Test KV shift
|
||||
id: test_kv_shift
|
||||
run: |
|
||||
./build/bin/test-llama-archs --kv-shift
|
||||
|
||||
- name: Test fusion
|
||||
id: test_fusion
|
||||
run: |
|
||||
@@ -300,11 +293,6 @@ jobs:
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
|
||||
|
||||
- name: Test KV shift
|
||||
id: test_kv_shift
|
||||
run: |
|
||||
./build/bin/test-llama-archs --kv-shift
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
name: "PR video demo label"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, reopened, synchronize]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
video-demo-label:
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Add or remove the video demo label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
# passed through env, so the body is only matched, never run, and is safe to read from a fork
|
||||
BODY: ${{ github.event.pull_request.body }}
|
||||
HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
|
||||
run: |
|
||||
# sections written by demo:pr (see ui/apps/chat/demo/cli.ts): the managed comment
|
||||
# markers, or a preview linking to a demo video uploaded for this PR
|
||||
has_demo=false
|
||||
if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
|
||||
grep -qE "/resolve/videos/pr/$PR_NUMBER/[0-9a-f]{10}(-[0-9]+)?\.mp4" <<< "$BODY"; then
|
||||
has_demo=true
|
||||
fi
|
||||
|
||||
if [ "$has_demo" = true ] && [ "$HAS_LABEL" = false ]; then
|
||||
gh pr edit "$PR_URL" --add-label 'video demo'
|
||||
elif [ "$has_demo" = false ] && [ "$HAS_LABEL" = true ]; then
|
||||
gh pr edit "$PR_URL" --remove-label 'video demo'
|
||||
fi
|
||||
@@ -24,7 +24,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
python-check-requirements:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
name: check-requirements
|
||||
steps:
|
||||
- name: Check out source repository
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.100
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.88
|
||||
# - name: Type-check with Pyright
|
||||
# uses: jakebailey/pyright-action@v2
|
||||
# with:
|
||||
|
||||
@@ -171,14 +171,12 @@ jobs:
|
||||
**Linux:**
|
||||
- [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
|
||||
- [Ubuntu 24.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz)
|
||||
- [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz)
|
||||
- [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
|
||||
- [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
|
||||
- [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
|
||||
- [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-2026.4.1-x64.tar.gz)
|
||||
- [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
|
||||
@@ -194,8 +192,8 @@ jobs:
|
||||
- [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
|
||||
- [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
|
||||
- [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-x64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-x64.zip)
|
||||
- [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-arm64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-arm64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
|
||||
- [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
|
||||
- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
|
||||
- [Windows arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-arm64.zip)
|
||||
- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-2026.4.1-x64.zip)
|
||||
|
||||
+37
-112
@@ -130,7 +130,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -180,8 +180,6 @@ jobs:
|
||||
os: ubuntu-24.04-arm
|
||||
- build: 's390x'
|
||||
os: ubuntu-24.04-s390x
|
||||
- build: '26.04-s390x'
|
||||
os: ubuntu-26.04-s390x
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@@ -196,13 +194,13 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libssl-dev
|
||||
sudo apt-get install build-essential libssl-dev
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
|
||||
@@ -212,7 +210,7 @@ jobs:
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: ccache
|
||||
if: ${{ !contains(matrix.os, 's390x') }}
|
||||
if: ${{ matrix.build != 's390x' }}
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
key: release-${{ matrix.os }}-cpu
|
||||
@@ -244,7 +242,7 @@ jobs:
|
||||
archive: false
|
||||
|
||||
- name: ccache-clear
|
||||
if: ${{ !contains(matrix.os, 's390x') }}
|
||||
if: ${{ matrix.build != 's390x' }}
|
||||
uses: ./.github/actions/ccache-clear
|
||||
with:
|
||||
key: release-${{ matrix.os }}-cpu
|
||||
@@ -274,7 +272,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -345,14 +343,14 @@ jobs:
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- build: 'x64'
|
||||
os: ubuntu-24.04
|
||||
cuda: '13.5.0'
|
||||
label: '13.5'
|
||||
defines: ''
|
||||
cuda: '13.4.1'
|
||||
label: '13.4'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- build: 'arm64'
|
||||
os: ubuntu-24.04-arm
|
||||
cuda: '13.5.0'
|
||||
label: '13.5'
|
||||
defines: ''
|
||||
cuda: '13.4.1'
|
||||
label: '13.4'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04
|
||||
@@ -381,7 +379,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -463,7 +461,7 @@ jobs:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
env:
|
||||
NDK_VERSION: "29.0.14206865"
|
||||
@@ -479,7 +477,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
@@ -548,7 +546,7 @@ jobs:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8'
|
||||
|
||||
defaults:
|
||||
@@ -571,7 +569,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -601,7 +599,7 @@ jobs:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8'
|
||||
|
||||
defaults:
|
||||
@@ -624,7 +622,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -672,7 +670,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -894,17 +892,12 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Install Ninja
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -975,11 +968,6 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
max-size: "1G"
|
||||
|
||||
- name: Tune ccache for ROCm
|
||||
run: |
|
||||
# since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
# - name: Cache ROCm Installation
|
||||
# id: cache-rocm
|
||||
# uses: actions/cache@v5
|
||||
@@ -1151,11 +1139,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
# TODO: these jobs need to use llvm toolchain in order to utilize the ccache
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -1221,12 +1204,12 @@ jobs:
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.5'
|
||||
- cuda: '13.4'
|
||||
arch: x64
|
||||
defines: ''
|
||||
- cuda: '13.5'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.4'
|
||||
arch: arm64
|
||||
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -1249,7 +1232,6 @@ jobs:
|
||||
with:
|
||||
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
|
||||
evict-old-files: 1d
|
||||
max-size: "1G"
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -1318,14 +1300,14 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
|
||||
WINDOWS_DNNL: intel.oneapi.win.dnnl
|
||||
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -1374,12 +1356,11 @@ jobs:
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_def.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_avx2.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_intel_thread.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/libimalloc.dll" ./build/bin
|
||||
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader_shared.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin
|
||||
ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true)
|
||||
if [ -n "$ZE_LOADER_DLL" ]; then
|
||||
@@ -1433,8 +1414,8 @@ jobs:
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
LEVEL_ZERO_VERSION: "1.34.0"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
steps:
|
||||
@@ -1449,12 +1430,12 @@ jobs:
|
||||
run: |
|
||||
cd /tmp
|
||||
echo "Download oneAPI Deep Learning Essentials installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
echo "Install oneAPI Deep Learning Essentials components"
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
echo "Download oneDNN installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh
|
||||
echo "Install oneDNN components"
|
||||
sudo bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
|
||||
@@ -1470,7 +1451,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1542,7 +1523,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Free up disk space
|
||||
uses: ggml-org/free-disk-space@v1.3.1
|
||||
@@ -1564,8 +1545,6 @@ jobs:
|
||||
# builds hit.
|
||||
ccache --set-config=compiler_check=content
|
||||
ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
|
||||
# since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1640,60 +1619,6 @@ jobs:
|
||||
with:
|
||||
key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
|
||||
|
||||
ubuntu-26-zdnn:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download UI build
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential libssl-dev libzdnn-dev
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
|
||||
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
|
||||
-DGGML_BACKEND_DL=ON \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_VXE=ON \
|
||||
-DGGML_ZDNN=ON \
|
||||
-DCMAKE_C_FLAGS="-march=arch15" \
|
||||
-DCMAKE_CXX_FLAGS="-march=arch15" \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
${{ env.CMAKE_ARGS }}
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Pack artifacts
|
||||
id: pack_artifacts
|
||||
run: |
|
||||
cp LICENSE ./build/bin/
|
||||
tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz
|
||||
archive: false
|
||||
|
||||
ios-xcode:
|
||||
needs: [check-release]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
@@ -42,8 +42,6 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
LLAMA_ARG_LOG_VERBOSITY: 10
|
||||
# [TAG_MTMD_NO_REALLOC] must match -DGGML_SCHED_NO_REALLOC=ON below, skips tests that are known to realloc
|
||||
GGML_SCHED_NO_REALLOC: ON
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
|
||||
@@ -39,8 +39,6 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
LLAMA_ARG_LOG_VERBOSITY: 10
|
||||
# [TAG_MTMD_NO_REALLOC] must match -DGGML_SCHED_NO_REALLOC=ON below, skips tests that are known to realloc
|
||||
GGML_SCHED_NO_REALLOC: ON
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
name: Server (speculative decoding)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
spec_methods:
|
||||
description: 'Speculative methods to test'
|
||||
required: true
|
||||
default: 'compact'
|
||||
type: choice
|
||||
options:
|
||||
- compact
|
||||
- nightly
|
||||
- all
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
server-speculative:
|
||||
runs-on: [self-hosted, Linux, gb10, perf]
|
||||
container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y cmake libssl-dev python3 python3-venv python3-pip
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: Python setup
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: ${{ github.event.inputs.spec_methods || 'compact' }}
|
||||
# pull requests from forks do not get repository variables
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || '/mnt/cache/spec-models' }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
|
||||
@@ -59,7 +59,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
ubuntu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Dependencies
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: server-ubuntu-26.04-arm
|
||||
key: server-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: server-ubuntu-26.04-arm
|
||||
key: server-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
@@ -175,11 +175,6 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -23,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Build application
|
||||
run: npm run build
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
retention-days: 1
|
||||
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-slim
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -37,26 +37,26 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Build application
|
||||
env:
|
||||
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
|
||||
run: npm run build
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run PWA unit tests (versioned build output)
|
||||
run: npx vitest --project=unit --run tests/unit/pwa.spec.ts
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
retention-days: 1
|
||||
@@ -1,183 +0,0 @@
|
||||
name: UI Chat Publish
|
||||
|
||||
# Publishes Llama Chat to npm: @ggml-org/llama-chat (ui/apps/chat, the app and its CLI) and
|
||||
# llama-chat (ui/apps/chat/alias, the short name for npx), always at the same version.
|
||||
#
|
||||
# The version in ui/apps/chat/package.json is the release, set by hand, as for @ggml-org/llama-ui
|
||||
# in ui-lib-publish.yml. A commit that reaches master with a version npm does not have builds the
|
||||
# app and publishes both packages; any other commit publishes nothing. Bump both package.json
|
||||
# files in the PR that should ship.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarballs with provenance. One-time setup, before this can publish: on
|
||||
# each of the two packages, Publishing settings, Trusted Publisher, GitHub Actions, with
|
||||
# organization ggml-org, repository llama.cpp, workflow file ui-chat-publish.yml and Environment
|
||||
# left blank. Until then the publish step fails on master; re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/package.json'
|
||||
]
|
||||
# a dry run for PRs that change what goes into the packages
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/**',
|
||||
'ui/apps/chat/bin/**',
|
||||
'ui/apps/chat/cli/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
APP_DIR: ui/apps/chat
|
||||
ALIAS_DIR: ui/apps/chat/alias
|
||||
VERSION_CHECK: ui/lib/ui/scripts/npm-version-check.mjs
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-chat and llama-chat
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Build the app
|
||||
# the same build.json version as the UI that llama-server embeds, for the app's update check.
|
||||
# This folder's .npmrc sets ignore-scripts, so npm never builds it: this step does.
|
||||
run: LLAMA_BUILD_NUMBER="b$(git rev-list --count HEAD)" npm run build
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Test the CLI and the alias version
|
||||
run: npx vitest --project=unit --run tests/unit/cli.test.ts
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Check what goes into the package
|
||||
run: |
|
||||
npm pack --dry-run --json > "$RUNNER_TEMP/pack.json"
|
||||
node -e '
|
||||
const [{ files }] = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
||||
const paths = files.map((file) => file.path);
|
||||
const maps = paths.filter((path) => path.endsWith(".map"));
|
||||
if (!paths.includes("dist/index.html")) {
|
||||
console.error("dist/index.html is not in the package: nothing would be served");
|
||||
process.exit(1);
|
||||
}
|
||||
if (maps.length > 0) {
|
||||
console.error(`source maps in the package: ${maps.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${paths.length} files, no source maps`);
|
||||
' "$RUNNER_TEMP/pack.json"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Compare @ggml-org/llama-chat with npm
|
||||
id: app
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$APP_DIR"
|
||||
|
||||
- name: Compare llama-chat with npm
|
||||
id: alias
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$ALIAS_DIR"
|
||||
|
||||
# master only: npm's trusted publisher matches the repository and the workflow file, not the
|
||||
# branch, so this condition is what keeps a manual run on a PR branch from publishing.
|
||||
# The app goes first, since the alias depends on it at the same version.
|
||||
- name: Publish @ggml-org/llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.app.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.app.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Publish llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.ALIAS_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# values reach the shell through env, so a name from a PR's package.json cannot inject here
|
||||
env:
|
||||
APP: ${{ steps.app.outputs.name }}@${{ steps.app.outputs.version }}
|
||||
APP_VERDICT: ${{ steps.app.outputs.verdict || 'not reached' }}
|
||||
APP_NOTE: ${{ steps.app.outputs.note }}
|
||||
ALIAS: ${{ steps.alias.outputs.name }}@${{ steps.alias.outputs.version }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
{
|
||||
echo "### Llama Chat on npm"
|
||||
echo
|
||||
echo "| package | verdict | note |"
|
||||
echo "| --- | --- | --- |"
|
||||
echo "| \`$APP\` | \`$APP_VERDICT\` | $APP_NOTE |"
|
||||
echo "| \`$ALIAS\` | \`$ALIAS_VERDICT\` | $ALIAS_NOTE |"
|
||||
echo
|
||||
if [ "$EVENT" = "pull_request" ] || [ "$DRY" = "true" ]; then echo "Dry run: nothing was published."; fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
# a version going backwards is a mistake; an unchanged version is a package.json edit that
|
||||
# is not a release, such as a dependency update
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
env:
|
||||
VERDICTS: ${{ steps.app.outputs.verdict }} ${{ steps.alias.outputs.verdict }}
|
||||
run: |
|
||||
case " $VERDICTS " in
|
||||
*" stale "*)
|
||||
echo "::error::a version in package.json is older than the one on npm"
|
||||
exit 1 ;;
|
||||
esac
|
||||
@@ -1,230 +0,0 @@
|
||||
name: UI Lib Publish
|
||||
|
||||
# Publishes @ggml-org/llama-ui, the library in ui/lib/ui, to npm.
|
||||
#
|
||||
# The version in ui/lib/ui/package.json is the release. A commit that reaches master with a
|
||||
# version npm does not have builds the library and publishes it; a commit that changes the
|
||||
# library without bumping it publishes nothing and says so on the run, which is the drift that
|
||||
# let 0.3.0 sit a day behind master. Bump the version in the PR whose change should ship, and
|
||||
# this workflow does the rest.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarball with a provenance attestation. release-publish.yml has done
|
||||
# the same exchange on every push to master since it landed, its actions/attest step being the
|
||||
# proof that this repository can be issued a token.
|
||||
#
|
||||
# One-time setup, before this can publish: on the package, Publishing settings, Trusted
|
||||
# Publisher, GitHub Actions, with organization ggml-org, repository llama.cpp, workflow file
|
||||
# ui-lib-publish.yml, and Environment left blank - this job names no environment, since creating
|
||||
# one needs repository admin. Merging this workflow before that exists leaves the publish step
|
||||
# red on master: the release is late rather than wrong, and re-running the job publishes it.
|
||||
#
|
||||
# The unscoped llama-ui is an alias: once the scoped package is out, the same build is renamed
|
||||
# and published again at the same version, so the two never drift. It is a package of its own
|
||||
# on npm, and needs the same trusted publisher set on it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
# the advisory half, so a library change without a bump is annotated on the PR rather than
|
||||
# discovered after the merge: the path filter keeps it off PRs that touch only Llama Chat
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back. GitHub holds
|
||||
# one pending run per group, so a newer push replaces an older queued one - no release is lost,
|
||||
# as each run reads the version from its own commit.
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
LIB_DIR: ui/lib/ui
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-ui
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
npm -v
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Build the package and lint it
|
||||
run: npm run prepack
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the version with npm
|
||||
id: version
|
||||
# on a PR this is advisory and must not fail the job: a registry outage is not the
|
||||
# contributor's mistake. On master the publish decision rests on the answer, so it fails.
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Note a library change that ships no new version
|
||||
if: github.event_name == 'pull_request' && always() && steps.version.outcome == 'success' && steps.version.outputs.should_publish != 'true'
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
run: echo "::warning::$NOTE"
|
||||
|
||||
- name: Publish
|
||||
# master only. npm's trusted publisher matches on the repository and the workflow file,
|
||||
# not the branch, so this condition is the only thing stopping a manual run from a PR
|
||||
# branch - one that anyone with write access can start - from publishing code that no
|
||||
# one reviewed. A dispatch on master remains the way to re-run a release.
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
# --ignore-scripts publishes the tarball prepack built and publint passed rather than
|
||||
# rebuilding it here; --provenance is implied by trusted publishing and named to be
|
||||
# explicit. A package published with nothing in it cannot be taken back, so refuse first.
|
||||
env:
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Rename the build to the unscoped alias
|
||||
# after the scoped publish, so nothing above sees the changed name. Not on a PR: the
|
||||
# advisory check there is about the version, which the alias shares.
|
||||
if: github.event_name != 'pull_request'
|
||||
run: node scripts/alias-package.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the alias with npm
|
||||
id: alias
|
||||
if: github.event_name != 'pull_request'
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Publish the alias
|
||||
# its own verdict rather than the scoped one's, so an alias that fell behind catches up
|
||||
# on the next run instead of waiting for a bump
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# every value reaches the shell through env, not through the script, so a package name
|
||||
# from a PR's package.json cannot inject into this step
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name || '@ggml-org/llama-ui' }}
|
||||
VERSION: ${{ steps.version.outputs.version || 'unknown' }}
|
||||
LATEST: ${{ steps.version.outputs.latest }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict || 'not reached' }}
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
SHOULD: ${{ steps.version.outputs.should_publish }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
ALIAS_LATEST: ${{ steps.alias.outputs.latest }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
ALIAS_SHOULD: ${{ steps.alias.outputs.should_publish }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo "### $NAME $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
[ "$EVENT" = "pull_request" ] && exit 0
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$ALIAS_SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo
|
||||
echo "### llama-ui (alias) $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${ALIAS_LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$ALIAS_VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $ALIAS_NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
if: always() && github.event_name != 'pull_request' && steps.version.outcome == 'success'
|
||||
# a version going backwards is a mistake; a version not bumped is usually one too, but it
|
||||
# is also what a stories-only commit looks like, so that only fails the run when the
|
||||
# repository variable UI_LIB_PUBLISH_FAIL_ON_UNBUMPED is set to true
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict }}
|
||||
FAIL_ON_UNBUMPED: ${{ vars.UI_LIB_PUBLISH_FAIL_ON_UNBUMPED }}
|
||||
run: |
|
||||
case "$VERDICT" in
|
||||
stale)
|
||||
echo "::error::$NOTE"
|
||||
exit 1 ;;
|
||||
published)
|
||||
if [ "$FAIL_ON_UNBUMPED" = "true" ]; then
|
||||
echo "::error::$NOTE"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::$NOTE" ;;
|
||||
esac
|
||||
@@ -33,15 +33,15 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
tar -czf dist.tar.gz -C ui/apps/chat/dist .
|
||||
tar -czf dist.tar.gz -C tools/ui/dist .
|
||||
sha256sum dist.tar.gz > dist.tar.gz.sha256
|
||||
mv dist.tar.gz dist.tar.gz.sha256 ui/apps/chat/dist/
|
||||
mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: pip install -U huggingface_hub
|
||||
@@ -52,12 +52,12 @@ jobs:
|
||||
- name: Sync built files to Hugging Face bucket (version tag)
|
||||
run: |
|
||||
# Upload the built files to the Hugging Face bucket under the release version
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
|
||||
- name: Sync built files to Hugging Face bucket (latest)
|
||||
run: |
|
||||
# Also upload to the 'latest' directory for fallback downloads
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
|
||||
- name: Verify upload
|
||||
run: |
|
||||
|
||||
@@ -17,9 +17,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'ui/lib/voice/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -27,9 +25,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'ui/lib/voice/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -71,33 +67,33 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Unit tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -113,25 +109,25 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run E2E tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
@@ -1,84 +0,0 @@
|
||||
name: UI Storybook
|
||||
|
||||
# Builds Llama UI and Llama Chat's Storybook (ui/apps/chat) and publishes it to its Hugging Face Space
|
||||
# whenever the UI changes on master, so the Space always shows the latest build.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-storybook.yml',
|
||||
'ui/apps/chat/**',
|
||||
'ui/lib/ui/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# only the newest build matters: a newer push replaces a deploy still in progress
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to Hugging Face Spaces
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
env:
|
||||
HF_SPACE: ${{ vars.HF_SPACE_UI_STORYBOOK || 'ggml-org/llama-ui-storybook' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Write the Space card
|
||||
run: |
|
||||
cat > ui/apps/chat/storybook-static/README.md << EOF
|
||||
---
|
||||
title: Llama UI Storybook
|
||||
emoji: 🦙
|
||||
colorFrom: gray
|
||||
colorTo: gray
|
||||
sdk: static
|
||||
pinned: false
|
||||
---
|
||||
|
||||
The Storybook of Llama UI and Llama Chat, built from [ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp) (\`ui/apps/chat\`) at ${GITHUB_SHA}.
|
||||
EOF
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-venv
|
||||
python3 -m venv .venv-hf
|
||||
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
|
||||
|
||||
- name: Upload to the Space
|
||||
env:
|
||||
# a token with write access to the Space; HF_TOKEN_UI_STATIC_OUTPUT if that one has it
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STORYBOOK || secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
run: |
|
||||
# --delete '*' removes files the new build no longer has; the Space's .gitattributes is always kept
|
||||
.venv-hf/bin/hf upload "$HF_SPACE" ui/apps/chat/storybook-static . \
|
||||
--repo-type space \
|
||||
--delete '*' \
|
||||
--commit-message "Storybook from ggml-org/llama.cpp@${GITHUB_SHA::10}"
|
||||
+17
-91
@@ -13,8 +13,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -22,8 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -68,118 +66,46 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Check DESIGN.md is current with the tokens
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check:design
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Unit tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
lib-checks:
|
||||
name: Library checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps chromium
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
# the unit tests and the Storybook stories (ui/lib/ui/tests/stories), which run as tests in Chromium
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
voice-checks:
|
||||
name: Voice checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/voice
|
||||
working-directory: tools/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -198,38 +124,38 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Download built UI artifacts (reuses ui-build)
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run E2E tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
@@ -12,7 +12,7 @@ permissions:
|
||||
jobs:
|
||||
update:
|
||||
name: Update Winget Package
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
if: github.repository_owner == 'ggml-org'
|
||||
|
||||
steps:
|
||||
|
||||
+4
-7
@@ -3,8 +3,8 @@ project("llama.cpp" C CXX)
|
||||
include(CheckIncludeFileCXX)
|
||||
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 1)
|
||||
set(LLAMA_VERSION_MINOR 0)
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 7)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
@@ -103,7 +103,7 @@ endif()
|
||||
|
||||
# subprocess spawning isn't a supported/sandbox-friendly operation on mobile OSes or in WASM
|
||||
if (CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android" OR ANDROID
|
||||
OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN OR CMAKE_SYSTEM_NAME STREQUAL "WASI")
|
||||
OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN)
|
||||
set(LLAMA_SUBPROCESS_DEFAULT OFF)
|
||||
else()
|
||||
set(LLAMA_SUBPROCESS_DEFAULT ON)
|
||||
@@ -134,7 +134,7 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
|
||||
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_UI "llama: build Llama Chat, the embedded chat app, for server" OFF)
|
||||
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
|
||||
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
|
||||
option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)
|
||||
|
||||
@@ -150,7 +150,6 @@ option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured
|
||||
# Required for relocatable CMake package
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/win-app.cmake)
|
||||
|
||||
if (NOT DEFINED LLAMA_BUILD_NUMBER)
|
||||
set(LLAMA_BUILD_NUMBER ${BUILD_NUMBER})
|
||||
@@ -270,8 +269,6 @@ if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
|
||||
add_subdirectory(tools/mtmd)
|
||||
endif()
|
||||
|
||||
llama_apply_app_manifests()
|
||||
|
||||
#
|
||||
# install
|
||||
#
|
||||
|
||||
+30
-24
@@ -2,29 +2,20 @@
|
||||
# multiple collaborators per item can be specified
|
||||
#
|
||||
# ggml-org/ci : CISC, danbev, ggerganov, netrunnereve, ngxson, taronaeo
|
||||
# ggml-org/ggml-base : aendk, fairydreaming, ggerganov
|
||||
# ggml-org/ggml-cann : hipudding
|
||||
# ggml-org/ggml-cpu : angt, chaxu01, ggerganov, max-krasnyansky, taronaeo
|
||||
# ggml-org/ggml-cuda : am17an, gaugarg-nv, IMbackK, JohannesGaessler, ORippler, thelittlefireman, ynankani
|
||||
# ggml-org/ggml-et : marty1885
|
||||
# ggml-org/ggml-cuda : JohannesGaessler, am17an, IMbackK, ORippler
|
||||
# ggml-org/ggml-hexagon : lhez, max-krasnyansky
|
||||
# ggml-org/ggml-hip : IMbackK, thelittlefireman
|
||||
# ggml-org/ggml-metal : forforever73, ggerganov, jhen0409, masterFoad
|
||||
# ggml-org/ggml-metal : ggerganov
|
||||
# ggml-org/ggml-opencl : lhez, max-krasnyansky
|
||||
# ggml-org/ggml-openvino : ravi9, wine99
|
||||
# ggml-org/ggml-riscv : xctan
|
||||
# ggml-org/ggml-rpc : am17an, rgerganov
|
||||
# ggml-org/ggml-sycl : arthw, Titaniumtown
|
||||
# ggml-org/ggml-rpc : rgerganov
|
||||
# ggml-org/ggml-sycl : arthw
|
||||
# ggml-org/ggml-vulkan : 0cc4m, jeffbolznv
|
||||
# ggml-org/ggml-webgpu : reeselevine, yomaytk
|
||||
# ggml-org/ggml-zdnn : AlekseiNikiforovIBM, Andreas-Krebbel, taronaeo
|
||||
# ggml-org/llama-base : am17an, CISC, ggerganov, ngxson, sanmai, ServeurpersoCom
|
||||
# ggml-org/llama-common : aldehir, angt, bartowski1182, danbev, ggerganov, ngxson, nikwen, pwilkin, ruixiang63, sanmai
|
||||
# ggml-org/llama-examples : ggerganov
|
||||
# ggml-org/llama-models : CISC, danbev, ggerganov, ngxson
|
||||
# ggml-org/ggml-zdnn : taronaeo
|
||||
# ggml-org/llama-common : ggerganov, aldehir, angt, danbev, ngxson, pwilkin
|
||||
# ggml-org/llama-mtmd : ngxson
|
||||
# ggml-org/llama-server : allozaur, angt, ggerganov, ngxson, ServeurpersoCom
|
||||
# ggml-org/llama-ui : allozaur
|
||||
# ggml-org/llama-server : ggerganov, ngxson, allozaur, angt, ServeurpersoCom
|
||||
# ggml-org/llama-ui : allozaur
|
||||
|
||||
/.devops/*.Dockerfile @ngxson
|
||||
/.github/actions/ @ggml-org/ci
|
||||
@@ -38,12 +29,28 @@
|
||||
/conversion/ @CISC
|
||||
/convert_*.py @CISC
|
||||
/docs/backend/snapdragon/ @ggml-org/ggml-hexagon
|
||||
/examples/ @ggml-org/llama-examples
|
||||
/examples/batched.swift/ @ggerganov
|
||||
/examples/batched/ @ggerganov
|
||||
/examples/convert-llama2c-to-ggml/ @ggerganov
|
||||
/examples/debug/ @danbev @pwilkin
|
||||
/examples/deprecation-warning/ @ggerganov
|
||||
/examples/diffusion/ @am17an
|
||||
/examples/embedding/ @ggerganov
|
||||
/examples/eval-callback/ @ggerganov
|
||||
/examples/export-docs/ @ggerganov
|
||||
/examples/gen-docs/ @ggerganov
|
||||
/examples/gguf/ @ggerganov
|
||||
/examples/llama.android/ @ggerganov @hanyin-arm @naco-siren
|
||||
/examples/llama.swiftui/ @ggerganov
|
||||
/examples/llama.vim @ggerganov
|
||||
/examples/lookahead/ @ggerganov
|
||||
/examples/lookup/ @JohannesGaessler
|
||||
/examples/model-conversion/ @danbev
|
||||
/examples/parallel/ @ggerganov
|
||||
/examples/passkey/ @ggerganov
|
||||
/examples/retrieval/ @ggerganov
|
||||
/examples/speculative-simple/ @ggerganov
|
||||
/examples/speculative/ @ggerganov
|
||||
/ggml/cmake/ @ggerganov
|
||||
/ggml/include/ @ggerganov
|
||||
/ggml/src/ggml-backend-meta.cpp @JohannesGaessler
|
||||
@@ -55,7 +62,7 @@
|
||||
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
|
||||
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
|
||||
/ggml/src/ggml-hexagon/ @ggml-org/ggml-hexagon
|
||||
/ggml/src/ggml-hip/ @ggml-org/ggml-hip
|
||||
/ggml/src/ggml-hip/ @IMbackK
|
||||
/ggml/src/ggml-et/ @marty1885
|
||||
/ggml/src/ggml-impl.h @ggerganov
|
||||
/ggml/src/ggml-metal/ @ggml-org/ggml-metal
|
||||
@@ -69,7 +76,7 @@
|
||||
/ggml/src/ggml-virtgpu/ @kpouget
|
||||
/ggml/src/ggml-vulkan/ @ggml-org/ggml-vulkan
|
||||
/ggml/src/ggml-webgpu/ @ggml-org/ggml-webgpu
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
|
||||
/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar
|
||||
/ggml/src/ggml.c @ggerganov
|
||||
/ggml/src/ggml.cpp @ggerganov
|
||||
@@ -80,15 +87,14 @@
|
||||
/scripts/get* @ggerganov
|
||||
/scripts/sync* @ggerganov
|
||||
/scripts/snapdragon/ @ggml-org/ggml-hexagon
|
||||
/src/ @ggml-org/llama-base
|
||||
/src/ @ggerganov
|
||||
/src/llama-adapter.* @CISC
|
||||
/src/llama-arch.* @CISC
|
||||
/src/llama-chat.* @ngxson
|
||||
/src/llama-graph.* @CISC
|
||||
/src/llama-model.* @CISC
|
||||
/src/llama-vocab.* @CISC
|
||||
/src/llama-moe-cache.* @am17an
|
||||
/src/models/ @ggml-org/llama-models
|
||||
/src/models/ @CISC
|
||||
/tests/ @ggerganov
|
||||
/tests/test-chat.* @pwilkin
|
||||
/tests/test-server-component.cpp @ggml-org/llama-server
|
||||
@@ -102,7 +108,7 @@
|
||||
/tools/rpc/ @ggml-org/ggml-rpc
|
||||
/tools/server/* @ggml-org/llama-server # no subdir
|
||||
/tools/server/tests/ @ggml-org/llama-server
|
||||
/ui/ @ggml-org/llama-ui
|
||||
/tools/ui/ @ggml-org/llama-ui
|
||||
/tools/tokenize/ @ggerganov
|
||||
/tools/tts/ @ggerganov
|
||||
/vendor/ @ggerganov
|
||||
|
||||
@@ -16,8 +16,6 @@ The project differentiates between 3 levels of contributors:
|
||||
>
|
||||
> Detailed information regarding permissible and restricted uses of AI can be found in the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
### Requirements
|
||||
|
||||
If AI is used to generate any portion of the code, contributors must adhere to the following requirements:
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
@@ -26,16 +24,6 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
### Checklist
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> **If you use AI, do all of the following:**
|
||||
>
|
||||
> 1. **Make sure [AGENTS.md](AGENTS.md) is loaded into your agent's context.** Skipping this step is one of the most common reasons PRs get rejected.
|
||||
> 2. **Search for related discussions, PRs and issues** to get the broader context before writing any code.
|
||||
> 3. **Run a local review pass before pushing anything**, using the [`code-review`](skills/code-review/SKILL.md) skill, once the change is implemented. Fix what it finds.
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
# Pull requests (for contributors & collaborators)
|
||||
|
||||
@@ -4,29 +4,19 @@ Maintainer documentation for this private fork. Read it before you rebase, chang
|
||||
anything. The rebase and deploy procedure is in [docs/fork-rebase-runbook.md](docs/fork-rebase-runbook.md),
|
||||
which also holds the operations reference (pod access, files, recovery).
|
||||
|
||||
Status at the time of writing (2026-11-22):
|
||||
Status at the time of writing (2026-10-25):
|
||||
|
||||
- Production runs `f3397cded` on branch `fable5/vram-ondemand-rebase-20261121` (upstream `e6d624b9a`,
|
||||
757 upstream commits, plus the temporary upstream carries #32735 and #32740, F12, F13, the
|
||||
per-pass loop views for Inkling, and F14, the user-message batch-break merge), deployed 2026-11-22
|
||||
10:37 (F14 validated in its own window: 35B TTFT A/B, disk cache 15 PASS and the 35B legs; F14
|
||||
measured). Rollback pointer `/root/DEPLOY-ROLLBACK-commit-16.txt` = `b13c41355` (build copy
|
||||
`/root/arms/build-b13c41355`; config unchanged by this deploy); next free pointer 17. The previous
|
||||
pointer 15 = `3621025e8` needs its config backup `/root/config.yaml.bak-20261122-predeploy` too.
|
||||
`system_fingerprint` is `b<build number>-<short sha>` (currently `b12687-f3397cded`) and changes
|
||||
with each rebase or carry. The hand ports of the 2026-11-21 rebase are listed in the runbook
|
||||
(section 3, "2026-11-21").
|
||||
- Previous branch `fable5/vram-ondemand-rebase-20261024` on upstream `3f03b1772` (production
|
||||
`adc2aa3ec` until 2026-11-21), kept untouched as the safety net. The fork commits are
|
||||
- Branch `fable5/vram-ondemand-rebase-20261024` on upstream `3f03b1772`. The fork commits are
|
||||
`git log --oneline origin/master..HEAD`; that list includes documentation and script commits
|
||||
(this file among them), so count it rather than trusting a number written here. Section 2 maps
|
||||
every commit to a feature.
|
||||
- The disk prompt cache is ON in production (F11). `2fc9f9c35` was validated with the full
|
||||
`fork-validate.sh` set (37 PASS, 0 FAIL). Any later commit that touches compiled code needs the
|
||||
same validation before a redeploy.
|
||||
- Branch `fable5/disk-prompt-cache` added F11 (disk prompt cache) and a CPU_REPACK release fix (F4).
|
||||
A deploy of cb48bf224 was rolled back on 2026-10-25 (mmproj abort, F11); the fix (adc2aa3ec) passed
|
||||
the 35B run with the production command line on 2026-11-21 and is deployed.
|
||||
- Production runs `2fc9f9c35` (deployed 2026-10-25 13:01 UTC, `system_fingerprint`
|
||||
`b11906-2fc9f9c35`; rollback pointer `/root/DEPLOY-ROLLBACK-commit-10.txt` = `6c480f7e7`, the
|
||||
previous production). It was validated with the full `fork-validate.sh` set (37 PASS, 0 FAIL, runbook
|
||||
section 6). Any later commit that touches compiled code needs the same validation before a redeploy.
|
||||
- Branch `fable5/disk-prompt-cache` adds F11 (disk prompt cache) and a CPU_REPACK release fix (F4) on
|
||||
top. A deploy of cb48bf224 was rolled back on 2026-10-25 (mmproj abort, F11); production stays on
|
||||
`2fc9f9c35`. The fix is on the branch; the 35B run with the production command line is still due.
|
||||
- Commit hashes below refer to that branch. They change on every rebase; find a commit again by
|
||||
its subject line. The commit messages are detailed and are the primary record of each mechanism
|
||||
and measurement; read them with `git show`.
|
||||
@@ -53,10 +43,6 @@ Status at the time of writing (2026-11-22):
|
||||
- F9 Benchmark harness and validation script
|
||||
- F10 Page-cache drop after a copying load
|
||||
- F11 Disk prompt cache (save slots on shutdown, restore after a restart)
|
||||
- F12 Temporary upstream carries (open upstream PRs, drop when merged)
|
||||
- F13 Per-pass views of host weights in looped layers (inkling2_loop)
|
||||
- F14 User-message batch-break merge (one ubatch per short turn)
|
||||
- F15 AVX2 NVFP4 x Q8_0 dot product (Inkling-2-Mini CPU experts)
|
||||
4. Dead ends
|
||||
5. Configuration that matters
|
||||
6. Measurement methodology and traps
|
||||
@@ -138,11 +124,6 @@ Oldest first. "Feature" points at the section below.
|
||||
| dd851fd0c | llama: read a released KV/recurrent state from its host shadow | F4, F11 |
|
||||
| 2874dcb10 | server: disk prompt cache, save slots on shutdown and restore after a restart | F11 |
|
||||
| 2c6c517ba | scripts: fork-validate.sh --diskcache, the disk prompt cache | F9, F11 |
|
||||
| d2396e4e3 | upstream-carry #32735: vulkan : support non-contiguous src in SCALE (open upstream PR, temporary) | F12 |
|
||||
| 72da62cea | upstream-carry #32740: ggml: compact bias on the existing FLASH_ATTN_EXT for CUDA, ROCm and Vulkan (open upstream PR, temporary) | F12 |
|
||||
| 312c29c2b | llama: inkling2_loop - per-pass views of host weights for the later loop passes (deployed 2026-11-22 as `b13c41355`) | F13 |
|
||||
| 510aa6e09 | server: merge user-message batch breaks when the rest of the prompt fits in one ubatch (deployed 2026-11-22 as `f3397cded`) | F14 |
|
||||
| 78a204bb2 | ggml-cpu: faster AVX2 NVFP4 x Q8_0 dot product (two blocks per iteration, vector UE4M3 scales) (branch `fable5/nvfp4-avx2`, not deployed) | F15 |
|
||||
|
||||
Later documentation-only commits (FORK.md, README, runbook) are not listed; they carry no code.
|
||||
Everything down to `cce0ac1f0` is in production (`2fc9f9c35`, the documentation commit after it);
|
||||
@@ -1058,11 +1039,8 @@ Swapdown window (2026-10-25, started by the owner, `/root/arms/diskcache-window.
|
||||
incident above; `--cache-disk-path` on a scratch dir, arbiter env, an 18760-token pan-tadeusz chat;
|
||||
log `/root/arms/diskcache-test/q35.log`). `q35.py` now uses the exact production entry (`-hf`, so
|
||||
the mmproj loads, nothing extra). That run (`/root/arms/diskcache-window3.sh`, about 8 minutes under
|
||||
swapdown, which also covers the same-reuse-point identity and an image request on the 35B) was run
|
||||
2026-11-21 on adc2aa3ec (log `/root/arms/diskcache-test/q35-mm.log`) and passed all legs: 18.8k-token
|
||||
chat, prefill 84.6 s -> 4.7 s for turn 2 after the restart with identical output; save 557 MB in
|
||||
1.0 s, token free 0.34 s, peak RSS +430 MB; cold save identity; image answered and not saved. The
|
||||
table below is the earlier `--no-mmproj` run:
|
||||
swapdown, which also covers the same-reuse-point identity and an image request on the 35B) is
|
||||
ready and not run:
|
||||
|
||||
| leg | result |
|
||||
|---|---|
|
||||
@@ -1082,9 +1060,10 @@ Two anomalies in leg c, both explained:
|
||||
cold save and a second restore. Z == X, text and top-3 logprobs bit-identical, on CPU and on the
|
||||
GPU (`-ngl 99`); turn 2 and turn 2 again also agree there. The same check on the 35B needs the GPU
|
||||
to itself: `/root/arms/diskcache-window3.sh` (production command line, reference "turn 2 again" against
|
||||
leg c; about 8 minutes under swapdown). It ran 2026-11-21 and passed (same-reuse-point identity:
|
||||
identical output), so the difference above is the reuse point. It supersedes
|
||||
`diskcache-window2.sh`, whose `cmp.py` runs with `--no-mmproj`.
|
||||
leg c; about 8 minutes under swapdown) is ready and not run. It supersedes `diskcache-window2.sh`,
|
||||
whose `cmp.py` runs with `--no-mmproj`.
|
||||
Until it runs, the 35B difference is attributed to the reuse point (a different batch split on the
|
||||
GPU), not proven on that model.
|
||||
- Exit peak RSS +3055 MiB and token free in 1881 ms (leg a: +430 MiB / 328 ms): a test harness bug.
|
||||
The driver's fake waiter was `flock -s want.lock sleep 120`; killing flock left `sleep` holding
|
||||
want.lock (flock(1) keeps the lock fd open in its child). So c had a permanent waiter, slept right
|
||||
@@ -1093,534 +1072,11 @@ Two anomalies in leg c, both explained:
|
||||
arrived during it. The cache's own save then ran as a cold save (copy 305 ms, no extra copy). The
|
||||
driver now holds the lock in the process it kills, like `fork-validate.sh` does.
|
||||
|
||||
Configuration (applied in production 2026-11-21): `--cache-disk-path /root/prompt-cache
|
||||
--cache-disk-size 4096` appended to the `moe_offload_args` and `moe_offload_gemma` macros in
|
||||
`/root/config.yaml` (backup `/root/config.yaml.bak-diskcache-20261121`). The directory is created on
|
||||
first start. Verified in production: a 35B 4.6k-token chat was saved on eviction (278.6 MiB, copy
|
||||
126 ms, write 495 ms), restored after the reload (read 416 ms, restore 33 ms), and turn 2 prefilled
|
||||
only 32 tokens; a gemma image request with the flags on works and is not cached. `/root` had 11-13 GB free on 2026-10-25, and the 1 GiB free-space floor still
|
||||
Recommended configuration (not applied): append `--cache-disk-path /root/prompt-cache
|
||||
--cache-disk-size 4096` to the `moe_offload_args` and `moe_offload_gemma` macros. The directory is
|
||||
created on first start. `/root` had 11-13 GB free on 2026-10-25, and the 1 GiB free-space floor still
|
||||
applies. Models only pick up the flags when they restart.
|
||||
|
||||
### F12. Temporary upstream carries (open upstream PRs, drop when merged)
|
||||
|
||||
**What and why.** Open upstream PRs carried unchanged until upstream merges them. Each carry is one
|
||||
commit whose subject starts with `upstream-carry #<PR>:`; the author is the PR author, and the body
|
||||
names the PR URL and the PR head that was carried. List them with
|
||||
`git log --oneline --grep '^upstream-carry' origin/master..HEAD`. They are not fork features: never
|
||||
edit them in place, re-fetch the PR instead.
|
||||
|
||||
Carried since 2026-11-22 (both on upstream `e6d624b9a`, for Inkling-2-Mini, section 5):
|
||||
- #32735 `vulkan : support non-contiguous src in SCALE` (head `0ea4ffd00`): the Vulkan backend runs
|
||||
SCALE on a non-contiguous source instead of rejecting it.
|
||||
- #32740 `ggml: compact bias on the existing FLASH_ATTN_EXT for CUDA, ROCm and Vulkan` (head
|
||||
`808fd1c4e`, four PR commits squashed): Inkling's per-query bias table (`[n_bins, n_head,
|
||||
n_tokens]` plus an I32 bin index per key) is applied inside the flash-attention kernel. Before it,
|
||||
the graph gathered the bias into a dense per-head mask (`n_kv x n_tokens x n_head`, F32, added to
|
||||
the KQ mask and cast to F16) and passed that to FA, and the Inkling layers could not use the
|
||||
`kv_rows` cell lists (dense masks only). Vulkan
|
||||
binds the table and the indices in the two mask_opt descriptor slots, so `use_mask_opt` gets
|
||||
`!bias` (F2). Vulkan supports it only for F16 K/V, head size 64 or 128, with subgroup shuffle and
|
||||
vote (RX 580 RADV has both; Inkling is f16 KV, head size 128).
|
||||
|
||||
Measured on the RX 580 (2026-11-21, the previous PR head: `1fa93f902`, `d29f9d489`, `ff91e4f84`,
|
||||
without `808fd1c4e`; base `3bf6b55b3` vs base + both PRs; logs `/root/arms/pr-out/` on the pod):
|
||||
- Inkling pp2048 at ub1024: 91.6 -> 110.5 t/s. pp2048 at depth 16384 (ub512): 48.4 -> 73.0 t/s.
|
||||
- Inkling compute buffers at ctx 131072, ub512: Vulkan0 12918 -> 7173 MiB, Vulkan_Host
|
||||
12729 -> 398 MiB. At ctx 65536 the base build already reserved 15866 MiB of Vulkan0 compute (more
|
||||
than the card; the driver places the excess in GTT). With the carry: 6981 MiB.
|
||||
- No regression elsewhere: Qwen 35B pp2048 286.5 -> 287.9, pp2048 at 16k 208.0 -> 208.5;
|
||||
gemma-26B 249.4 both.
|
||||
- test-backend-ops on Vulkan0: SCALE 15/15, FLASH_ATTN_EXT 5693/5693.
|
||||
`808fd1c4e` (the PR head since 2026-11-22) changes the table layout from `[n_head, n_bins, n_tokens]`
|
||||
to `[n_bins, n_head, n_tokens]`, which removes one permute+cont from the Inkling graph.
|
||||
|
||||
Measured with the carried head (2026-11-22, candidate `3621025e8` vs production `3bf6b55b3`,
|
||||
production's Inkling command line with the nothink sampling, GPU to itself; driver
|
||||
`/root/arms/carry1122-ink-direct.py`, logs `/root/arms/carry1122-p2.log`, `-p3.log`):
|
||||
|
||||
| build, args | 8.2K-token prefill | 13.9K | 28.4K | process VRAM / GTT after load |
|
||||
|---|---|---|---|---|
|
||||
| production, ub512, ctx 65536 | 32.7 t/s | 31.6 | 29.5 | 7209 / 28113 MiB |
|
||||
| carry, ub1024, ctx 131072 | 43.7 | 42.5 | 42.5 | 7983 / 21835 MiB |
|
||||
| carry, ub1024, ctx 65536 | 46.8 | 46.9 | 46.0 | 7368 / 19629 MiB |
|
||||
|
||||
- Device memory wanted (`-lv 4` breakdown): production ub512/65536 19424 MiB (compute 15866); carry
|
||||
ub1024/131072 13388 MiB (model 1266, context 4419, compute 7702), host compute 798 MiB. Everything
|
||||
above the 8 GB card lands in GTT (system RAM over PCIe): about 5.2 GB at 131072. That costs about
|
||||
8% prefill against 65536, so production runs ub1024 with ctx 65536 (section 5) until the loop-layer
|
||||
experts move into VRAM, which needs that headroom.
|
||||
- Answers: a ~160-token first request on a fresh server (upstream #32778, all-'?' output) was
|
||||
correct on every build. Pan Tadeusz fragments of 8.2K and 13.9K tokens gave coherent, correct
|
||||
summaries on every build (upstream #32755 did not reproduce). The nothink entry
|
||||
(`--reasoning-effort none --reasoning-budget 0`) returns repeated
|
||||
`<|message_model|><|content_thinking|><|end_message|>` as content, with no tool call, for an image
|
||||
request, a tool call and a 28.4K-token weather log ("which station is in Record 5?"). This happens
|
||||
on every build, production `3bf6b55b3` with ub512 included, with and without
|
||||
`GGML_OP_OFFLOAD_MIN_BATCH=64`, so the carry did not cause it. The think entry gets all three
|
||||
right on both builds (image "Red", `get_weather({"city":"Warsaw"})`, 28.4K log "65" at 45.9 t/s
|
||||
in production). Short text prompts work on the nothink entry.
|
||||
- Nothink fix (2026-11-22, config only): the cause was `--reasoning-budget 0`. With effort `none`
|
||||
and no budget the model still writes a short reasoning block (100-550 characters) and then
|
||||
answers; a budget of 0 cut that block off at once and the model looped on the chat-structure
|
||||
tokens. The macro is now `--temp 1.0 --top-p 1.0 --top-k 0 --min-p 0.0 --reasoning-effort none`.
|
||||
Verified on production `3621025e8` (pod driver `/root/arms/overnight/nothink.py`, logs
|
||||
`nothink1.log`, `nothink2.log` there): short question, Polish poem, image ("Red"), tool call
|
||||
(`get_weather({"city":"Warsaw"})`) and a 29.4K-token Pan Tadeusz prompt (correct, 46.0 t/s) all
|
||||
OK. Effort `minimal` and a budget of 256 also work; the budget of 0 fails on all but the short
|
||||
question.
|
||||
|
||||
**Rebase hazards.**
|
||||
- At each rebase, check every carry first (runbook section 1): merged upstream -> drop the carry
|
||||
commit; still open -> compare `git diff <carried-head> <current PR head>` and re-carry the current
|
||||
head if it moved; closed unmerged -> decide with the owner (Inkling's prefill and memory depend on
|
||||
#32740, section 5).
|
||||
- #32740 conflicts with F2 on `use_mask_opt` in `ggml_vk_flash_attn`. The resolution: upstream's
|
||||
`!use_sparse`, the PR's `!bias &&`, the fork's GCN `HSK >= 256 || HSV >= 256` and its comment.
|
||||
When the PR is merged and the carry dropped, the same line conflicts again when F2 is re-applied
|
||||
on top; resolve it the same way.
|
||||
- #32740 changes `llama_kv_cache::set_input_relative_pos` (new `rows` argument) and the
|
||||
supports_op of every backend; a fork change in those places must follow it.
|
||||
|
||||
**Validation.** `test-backend-ops -b Vulkan0 -o SCALE` and `-o FLASH_ATTN_EXT` (check the case
|
||||
count), and the PR's own test `tests/test-flash-attn-indexed-bias` (build target
|
||||
`test-flash-attn-indexed-bias`; it runs every device, prints one line per case it ran and skips
|
||||
unsupported cases silently, so count the `Vulkan0:` lines: 16 f16 cases on the RX 580, max error
|
||||
about 1e-4. The 4 Q4_0 K/V cases run on no backend here, and the CPU backend rejects indexed bias
|
||||
(the graph then uses the dense mask), so it prints no CPU lines).
|
||||
Then an Inkling request through llama-swap.
|
||||
|
||||
### F13. Per-pass views of host weights in looped layers (inkling2_loop)
|
||||
|
||||
Branch `fable5/inkling-loop-views` (on top of the F12 carries), fast-forwarded into the production
|
||||
branch on 2026-11-22 (deploy and results below).
|
||||
|
||||
**What and why.** `inkling2_loop` (Inkling-2-Mini) runs the core layers 6..17 three times; the 48
|
||||
cache slots are 24 physical layers plus 2 x 12 repeats. `post_arch_tensors` makes every repeat slot
|
||||
share the tensors of its physical layer (`layers[il] = layers[phys]`). With the experts in host
|
||||
memory (`--n-cpu-moe 24`, `-ot exps=CPU`), `ggml_backend_sched` makes one device copy per (host
|
||||
tensor, backend) per graph, hashed by tensor pointer, at the first split that reads it, and keeps it
|
||||
allocated until the last reader (`ggml_backend_sched_split_graph`, pass 5 and the graph_copy input
|
||||
nodes). Two consequences:
|
||||
- Memory: the expert copies of all 12 loop layers are alive at the same time (about 547 MiB per
|
||||
layer on UD-Q4_K_XL, 6.5 GiB in total). That is most of the ~7 GiB Vulkan0 compute buffer in the
|
||||
F12 measurements.
|
||||
- Correctness (upstream issue #32755, "GGGG" output): the copy callback (`sched_copy_experts`,
|
||||
F1) runs once, for the first reader. When it reads the routing ids back (ubatches below the F1
|
||||
skip threshold of `4 * n_expert` ids, and at or above the Vulkan MUL_MAT_ID offload batch of 32
|
||||
tokens: 32-63 tokens on Inkling-2-Mini), it uploads only the experts pass 0 selected. Passes 1 and
|
||||
2 reuse that copy and read stale VRAM for every other expert. Ubatches of 64+ tokens are correct
|
||||
(F1 uploads every expert), ubatches below 32 run MUL_MAT_ID on the CPU (no copy).
|
||||
|
||||
**Where.** `src/llama-model.h` (`llama_model::post_load_tensors`, a new optional hook),
|
||||
`src/llama-model.cpp` (`llama_model_base::load_tensors` calls it on the no_alloc return and after
|
||||
`load_tensors_hadamard()`), `src/models/models.h` and `src/models/inkling2-loop.cpp`
|
||||
(`llama_model_inkling2_loop::post_load_tensors`, member `ctx_visits`). Commit 312c29c2b.
|
||||
|
||||
**How it works.** After the weights are loaded, for every slot of pass 1 and 2 (`il >= n_phys`), each
|
||||
tensor pointer in its `llama_layer` (walked as an array of pointers, as `llama_context::opt_init`
|
||||
does) that sits in a host buffer (`ggml_backend_buffer_is_host`) is replaced with
|
||||
`ggml_view_tensor` of it, named `<name>.visit<pass>`, initialized with `ggml_backend_view_init`
|
||||
(same buffer and data). On the no_alloc pass (memory fit) only `view->buffer` is set to the dummy
|
||||
buffer, so the scheduler still sees a WEIGHTS buffer. The view is a leaf (op NONE, `view_src` set),
|
||||
so the scheduler hashes it separately: each pass gets its own copy, freed after the last reader of
|
||||
that pass, and the copy callback sees the MUL_MAT_ID of that pass and its ids (`st.ids` differs, so
|
||||
F1 recomputes the expert set). The pointer-keyed weight maps of `llama_model` (`w_scales`,
|
||||
`prec_policy.prec_acc`/`prec_src1`, `hdmd.rot`) get entries for the views. The views live in
|
||||
`ctx_visits`, a context of the model outside `ctxs_bufs`. The startup log prints
|
||||
`post_load_tensors: N views of host weights for the loop passes after the first (X MiB, not copied)`.
|
||||
|
||||
Knob: `LLAMA_LOOP_HOST_VIEWS=0` keeps the shared tensors (the upstream behaviour), read once at load.
|
||||
Default on.
|
||||
|
||||
**Invariants that must hold.**
|
||||
- Only host buffers get views. Device weights stay shared (used in place, no copy, so no problem),
|
||||
and so do CPU extra buffers (CPU_REPACK is not a host buffer; its ops run on the CPU in place).
|
||||
- F4: views point only at host buffers, which `release_device_weights` never releases, and they
|
||||
are not in `ctxs_bufs`, so the shadows, `weight_release`, `memory_breakdown`, `n_tensors()` and
|
||||
`tensors_by_name` do not see them. A device weight is released and restored in place, and the
|
||||
loop slots keep pointing at the same tensor.
|
||||
- F1: the skip/readback decision is per copy and keys on the ids tensor of the first node of the
|
||||
split, so it stays per pass.
|
||||
- `llama_layer` must hold only `ggml_tensor *` members (a `static_assert` checks the size only).
|
||||
|
||||
**Cost.** Pass 1 and 2 now upload their own experts: per ubatch the host-to-device expert traffic
|
||||
goes from 24 to 48 layer uploads (about +13 GiB per full ubatch on UD-Q4_K_XL). Measured where the
|
||||
old layout still fits in VRAM (llama-bench pp2048 at depth 0, `-ncmoe 24`, B C C B in one window):
|
||||
views 107.10 / 107.53 t/s, off 110.80 / 111.53, so -3.5%. Decode is unaffected (single-token
|
||||
MUL_MAT_ID runs on the CPU). In the server at 16k-65k context the old layout does not fit (7 GiB of
|
||||
compute buffer), spills into GTT, and the views win about 1.6x at prefill (Measured, below). The
|
||||
freed ~6 GiB of VRAM can hold experts instead: an expert tensor of a loop layer kept on the GPU
|
||||
saves 3 uploads per ubatch and 3 CPU expert passes per token, a non-loop one saves 1, so loop layers
|
||||
are the first to move to the GPU (`-ot` with the loop layers left out of the CPU pattern; placement
|
||||
guide below).
|
||||
|
||||
**Measured (2026-11-22, pod, GPU to itself, build `b13c41355` = 312c29c2b + docs; drivers and logs in
|
||||
`/root/arms/overnight/`: `f13.py`, `cmp.py`, `kld48.sh`, `fit.py`, `sweep.sh`, `STATUS.md`).**
|
||||
- Memory, server with the production command line at ctx 16384, `-lv 4`: Vulkan0 compute buffer
|
||||
1426 MiB with views, 7030 MiB without. Process VRAM after load: views 4179 MiB, off 7727, A
|
||||
(loop experts on the GPU, no views) 7879 (A wants 10089 MiB and spills).
|
||||
- Prefill, server, 7224-token raw prompt at ctx 16384: views 92.4 t/s, off 57.0, A 46.5 (production
|
||||
at ctx 65536 without views: about 46).
|
||||
- Correctness. First-token top-5 logprobs (`f13.py`, `n_probs 5`, knob 32 so the 32-63 window is
|
||||
live): with the tail at or above 64 or below 32 tokens (N = 20, 100, 7368) views on and off are
|
||||
identical (max difference 0.000); in the window (N = 40, 63, 1072, 7224) they differ (0.04-0.16).
|
||||
That test cannot say which arm is right: arm A differs from both by up to 0.15 even at the
|
||||
controls (different placement, different kernels), and at 32-63 tokens x 8 experts pass 0's
|
||||
union already covers most of the 128 experts, so the stale copy is mostly correct data. The
|
||||
decisive test is `kld48.sh`: `llama-perplexity -c 48 -b 48 -ub 48`, 30 chunks of Pan Tadeusz, so
|
||||
every chunk is one 48-token ubatch, KL divergence against a base with views on and knob 64 (the
|
||||
tail MUL_MAT_ID on the CPU, correct by construction):
|
||||
|
||||
| arm | mean KLD | max KLD | same top token | RMS dp |
|
||||
|---|---|---|---|---|
|
||||
| views on, knob 32 (tail on the GPU) | 0.00314 | 0.087 | 97.4% | 1.35% |
|
||||
| views off, knob 32 (tail on the GPU) | 0.03163 | 0.347 | 92.2% | 3.71% |
|
||||
| views off, knob 64 | 0.00000 | - | 100% | 0.00% |
|
||||
|
||||
Views on is at the GPU-vs-CPU kernel noise level; views off is 10x worse (the #32755 mechanism,
|
||||
reproduced). So with F13 the `GGML_OP_OFFLOAD_MIN_BATCH=64` workaround is no longer needed.
|
||||
|
||||
**Placement guide (UD-Q4_K_XL, views on).** k = number of loop layers (6, 7, ...) whose experts stay
|
||||
on the GPU: `-ot 'blk[.](<the other 24-k layers>)[.]ffn_.*_exps=CPU'` instead of `--n-cpu-moe 24`
|
||||
(`/root/arms/overnight/mkot.py` and `cfgedit.py` write the pattern; `[.]` avoids backslashes in the
|
||||
YAML and in llama-swap's argument splitting). Each loop layer's experts take 547 MiB. Fit, from the
|
||||
`-lv 4` breakdown of a server with the production command line (mmproj on demand, so not loaded):
|
||||
|
||||
| ctx | k | Vulkan0 self (model + context + compute) | free | verdict |
|
||||
|---|---|---|---|---|
|
||||
| 32768 | 0 | 4136 (1266 + 1347 + 1522) | 3982 | fits |
|
||||
| 32768 | 6 | 7416 | 699 | fits |
|
||||
| 32768 | 7 | 7963 | 15 | too tight |
|
||||
| 65536 | 4 | 7246 (3453 + 2371 + 1421) | 868 | fits, chosen |
|
||||
| 65536 | 5 | 7793 | 128 | too tight for the mmproj |
|
||||
| 65536 | 6 | - | - | spills (GTT about 900 MiB above expectation) |
|
||||
| 131072 | 0 | 7784 (1266 + 4419 + 2098) | 134 | fits (before F13 it overcommitted 5.2 GB) |
|
||||
| 131072 | 2 | - | - | spills (about 1.2 GB) |
|
||||
|
||||
fdinfo `drm-memory-gtt` is not the spill test: it includes the pinned host weights
|
||||
(Vulkan_Host). Compare GTT(k) with GTT(0) - k x 547 MiB, or read the breakdown.
|
||||
|
||||
Speed, llama-bench, one invocation per metric (`sweep.sh`, `-fa 1 -t 6 -b 1024 -ub 1024`, knob 64,
|
||||
pp r1, tg r2):
|
||||
|
||||
| k | pp2048 @ d8192 | pp2048 @ d16384 | tg256 @ d4096 |
|
||||
|---|---|---|---|
|
||||
| 0 | 97.53 | 89.62 | 5.57 |
|
||||
| 4 | 102.19 | 93.90 | 6.45 |
|
||||
| 6 | 104.43 | 95.27 | 6.72 |
|
||||
| 7 | 105.90 | 96.90 | 7.25 |
|
||||
|
||||
Decode gains most (+16% at k=4, +30% at k=7): every loop layer on the GPU removes 3 of the 48 CPU
|
||||
expert passes per token. Prefill gains only 5-9%: at depth the attention and the remaining uploads
|
||||
dominate.
|
||||
|
||||
**Quants (2026-11-22, not adopted).** Unsloth's Mainline_Support UD-Q3_K_XL (13.6 GB) and UD-IQ3_XXS
|
||||
(11.7 GB) are in `/root/models/inkling/`. Quality: `llama-perplexity -c 2048 --chunks 4`, KL
|
||||
divergence against UD-Q4_K_XL logits (`kld.sh`), on a fixed Pan Tadeusz chunk
|
||||
(`/root/bench/entext/pt-chunk.txt`, characters 20000-100000) and wikitext-2 test:
|
||||
|
||||
| text | quant | PPL (Q4_K_XL) | PPL ratio | mean KLD | same top token |
|
||||
|---|---|---|---|---|---|
|
||||
| Pan Tadeusz | Q3_K_XL | 11.45 (11.30) | +1.4% | 0.0241 | 92.5% |
|
||||
| Pan Tadeusz | IQ3_XXS | 11.52 (11.30) | +1.9% | 0.0421 | 90.1% |
|
||||
| wikitext-2 | Q3_K_XL | 6.251 (6.230) | +0.3% | 0.0097 | 95.6% |
|
||||
| wikitext-2 | IQ3_XXS | 6.307 (6.230) | +1.2% | 0.0214 | 94.2% |
|
||||
|
||||
Fit at ctx 65536: UD-Q3_K_XL keeps the loop-layer experts at 547 MiB each (like Q4), so k=5 (867
|
||||
MiB free); UD-IQ3_XXS has 390 MiB per layer, k=7 (952 MiB free). Speed (separate invocations, so
|
||||
within the ~7% cross-run noise for prefill): Q3_K_XL k=4 105.23 / 6.83, k=6 107.31 / 7.48;
|
||||
IQ3_XXS k=4 105.59 / 6.44, k=7 108.48 / 7.86 (pp2048 @ d8192 / tg256 @ d4096; Q4_K_XL k=4
|
||||
102.19 / 6.45). The best case, IQ3_XXS at k=7, is +6% prefill and +22% decode (about 7.9 instead
|
||||
of 6.5 t/s) for a Polish KLD of 0.042; IQ3_XXS's CPU dot product is slower per byte (at k=4 it
|
||||
decodes no faster than Q4). Not worth the quality loss for the owner's Polish use.
|
||||
|
||||
**Production (deployed 2026-11-22, `b13c41355`, ctx 65536, k=4, no knob).** Through llama-swap
|
||||
(`/root/arms/overnight/post.py`, log `post.log`): Pan Tadeusz 9312 tokens 81.6 t/s prefill, +7958
|
||||
tokens 89.3 t/s, +17255 tokens (28K total) 78.0 t/s, decode 5.9-6.0 t/s, answers correct (before:
|
||||
about 46 t/s prefill). Think and nothink entries answer text, an image ("Red") and a tool call. A
|
||||
tiny request after the 4B task model held the GPU takes 1.3-2.4 s longer than a warm one (the
|
||||
arbiter restores 3.4 GiB of device weights). Disk cache save/restore works on Inkling (1329 tokens,
|
||||
410 MiB). The first request on a fresh process is slow (162 tokens 11.7 t/s prefill, 2.6 t/s decode)
|
||||
since the pipelines are compiled on first use (`--no-warmup`).
|
||||
|
||||
**Profile (2026-11-22, `perf record` + `gpu_busy_percent`, `prof.sh`, results in
|
||||
`/root/arms/overnight/prof/`).** Inkling k=4 prefill (pp2048 @ d4096, 107 t/s): GPU busy 77%, CPU
|
||||
time mostly `ggml_vk_wait_for_fence` (43%) and the kernel (11%, the uploads): GPU-bound. Decode
|
||||
(7.3 t/s): GPU busy 58%, CPU in the OpenMP barrier spin (libgomp 36%) and
|
||||
`ggml_vec_dot_nvfp4_q8_0` (21%): the UD-Q4_K_XL experts are NVFP4, and that CPU dot product is the
|
||||
decode hotspot. Qwen3.6-35B for comparison: prefill 263 t/s at GPU busy 52%, decode 13.8 t/s at
|
||||
35%, `vec_dot_q4_K_q8_K` 10%. `GGML_VK_PERF_LOGGER=1` aborts at the second split
|
||||
(`GGML_ASSERT(ctx->compute_ctx.expired())`, ggml-vulkan.cpp:15692), so it gives per-op GPU times
|
||||
only for the first split; not investigated.
|
||||
|
||||
**Failure modes and known limits.**
|
||||
- LoRA adapters look weights up by name (`llama_adapter_lora::get_weight`), so they do not apply to
|
||||
passes 1 and 2 with views on. Not used here; set `LLAMA_LOOP_HOST_VIEWS=0` for a LoRA on Inkling.
|
||||
- `--moe-cache-mib`: the cache binds expert tensors per slot (`llama_moe_cache_layer_experts` of
|
||||
`model.layers[il]`), so with views each loop slot gets its own binding and slot map and caches the
|
||||
same experts separately (before, the three slots collided on one binding key). Should not crash;
|
||||
not tested. The MoE cache is off in production.
|
||||
- Model saving is not supported for `inkling2_loop` (`llama_arch_supports_save`), so duplicate data
|
||||
in a saved file cannot happen.
|
||||
|
||||
**How to drop it.** If upstream fixes #32755 (for example per-reader copies or a copy-callback call for
|
||||
every reader in the scheduler, or real per-pass tensors in the loader), revert 312c29c2b; the hook has
|
||||
no other user. Check that upstream's fix also bounds the copy lifetime, otherwise the 6.5 GiB comes back.
|
||||
|
||||
**Rebase hazards.** `post_arch_tensors` and the loop slot layout in `inkling2-loop.cpp`;
|
||||
`llama_model_base::load_tensors` returns (a new return path must call the hook); new pointer-keyed
|
||||
weight maps in `llama_model` (they need a view entry like `w_scales`); non-pointer members in
|
||||
`llama_layer`; scheduler changes to how host weights are copied (`hv_tensor_copies`,
|
||||
`ggml_backend_sched_is_host_weight`).
|
||||
|
||||
**Validation (pod, GPU to itself in a swapdown window).** Build `llama-server` and `llama-bench`
|
||||
from this branch in a scratch worktree (runbook section 5). Use the Inkling command line from
|
||||
`inkling_args` in `/root/config.yaml` (with `--mmproj`) and change only the context to 16384 (so
|
||||
arm A fits) and the placement:
|
||||
- A, reference: replace `--n-cpu-moe 24` with `-ot 'blk\.([0-5]|1[89]|2[0-3])\.ffn_.*_exps=CPU'`
|
||||
(loop experts on the GPU, no copies for the loop; the views log line must not appear).
|
||||
- B, views on: `--n-cpu-moe 24`; the log line must show the views.
|
||||
- C, views off: B with `LLAMA_LOOP_HOST_VIEWS=0`.
|
||||
|
||||
For each arm send raw token prompts (`/tokenize` a long text, cut to N tokens, `/completion` with the
|
||||
token array, `temperature 0, top_k 1, seed 42, cache_prompt false, n_predict 32, n_probs 5`) at
|
||||
N = 40, 63, 1072, 6176, 7224 (tail 32-63 at ub1024), controls N = 100, 7368 (tail >= 64) and 20.
|
||||
Expected:
|
||||
- B == A in the first token's top-5 probabilities for every N >= 32 (prefill MUL_MAT_ID runs on the
|
||||
GPU in both arms with the same data). The generated text may drift later, since decode MUL_MAT_ID
|
||||
runs on the GPU in A and on the CPU in B.
|
||||
- B == C byte-identical for the controls (tail >= 64 or < 32).
|
||||
- C differs from A at the 32-63 tails (the bug); if it does not, the bug was not reproduced and
|
||||
B's result proves nothing.
|
||||
- Vulkan0 compute buffer (startup log): B about 6 GiB below C. A's model buffer carries the loop
|
||||
experts instead.
|
||||
Then `llama-bench -lm none -ngl 99 -ncmoe 24 -fa 1 -t 6 -b 1024 -ub 1024 -p 2048 -n 0 -r 3`, with and
|
||||
without `LLAMA_LOOP_HOST_VIEWS=0`, interleaved B C C B, medians (section 6), for the prefill cost
|
||||
above. After a deploy: an Inkling text request, an image request, and the same text request after a
|
||||
task-model call (arbiter handoff, F4), identical at temperature 0.
|
||||
|
||||
---
|
||||
|
||||
### F14. User-message batch-break merge (one ubatch per short turn)
|
||||
|
||||
Branch `fable5/userstart-split` (rebased onto the F13 production tip `166ec22e2`), fast-forwarded into
|
||||
the production branch on 2026-11-22 (deploy and results below). Knob: `LLAMA_USERSTART_MERGE=0`.
|
||||
|
||||
**What and why.** Upstream ends a prompt batch right before a user message so that a context
|
||||
checkpoint lands there (#22929 for the last user message, #24176 for every user message, #25420
|
||||
limited the earlier ones to `--checkpoint-min-step`). On a hybrid/recurrent model the recurrent state
|
||||
cannot be cut back, so a checkpoint is the only way to reuse a prefix; the one at the last user
|
||||
message is what a later request uses when it diverges inside that message (the user edits it, or the
|
||||
client injected per-request context into it, RAG/web search). Ordinary turns do not need it: Qwen's
|
||||
template drops the previous answer's reasoning, which moves the divergence to just after
|
||||
`<|im_start|>assistant\n`, below the end-minus-4 checkpoint (#20288), and that one is what turn N+1
|
||||
restores. The cost on this hardware: a turn processes `[previous answer | new user message + generation
|
||||
prompt]`. With the break that is two ubatches instead of one, and with every expert on the host each
|
||||
ubatch of >= 32 tokens (`GGML_OP_OFFLOAD_MIN_BATCH`) uploads the experts again: all of them from 128
|
||||
tokens (F1), about 64-96% of them at 32-127 tokens (routing readback), about 2.8 s per full upload on
|
||||
the 35B. The break also fires at the first user message of a fresh prompt (no checkpoints yet),
|
||||
which splits off the system prompt. Tool results count as user messages (Qwen opens them with
|
||||
`<|im_start|>user`, the detected delimiter).
|
||||
|
||||
**Where.** `tools/server/server-context.cpp`, `update_slots`, the prompt-batch loop:
|
||||
`merge_user_starts` before the loop and the `!merge_user_starts` term of the user-start break.
|
||||
Commit 510aa6e09.
|
||||
|
||||
**How it works.** When the rest of the prompt from the current batch start fits in one ubatch
|
||||
(`task.n_tokens() - prompt.n_tokens() <= n_ubatch`) and no media chunk was processed in this
|
||||
iteration, the user-start break is skipped. The batch then runs to the end-minus-4 break: one ubatch
|
||||
plus the 4-token tail, as for a prompt without user messages. Checkpoints left: the batch start
|
||||
(the near-prompt-end rule; the batch ends at prompt end minus 4, so it always applies; it supersedes
|
||||
the restored checkpoint at the same position, with the current task id, so the next request still
|
||||
has it) and the prompt end minus 4. Lost: the one at the last user message. A later request that
|
||||
diverges inside that message restores the batch-start checkpoint instead and re-processes the
|
||||
previous answer too, less than one ubatch, in the same single ubatch, so no extra upload. Prompts
|
||||
whose rest is longer than one ubatch keep the upstream breaks (there the break does not save the
|
||||
whole of a ubatch, and the fallback would not be bounded). `LLAMA_USERSTART_MERGE=0` restores the
|
||||
upstream behaviour everywhere.
|
||||
|
||||
**Invariants that must hold.**
|
||||
- The merge only applies when the rest fits in one ubatch: that bounds the extra re-processing on a
|
||||
divergence inside the last user message to less than `n_ubatch` tokens.
|
||||
- The batch-start checkpoint must still be created (near-prompt-end rule, `do_checkpoint` not cleared
|
||||
by `has_mtmd`); hence the `!has_mtmd` condition.
|
||||
- It only changes where batches end, never which tokens are processed, so outputs differ from an
|
||||
unmerged run only by batch-composition rounding.
|
||||
|
||||
**Measured (2026-11-22, pod, GPU to itself in one swapdown window, build `f3397cded`; driver
|
||||
`/root/arms/usw_ab.py`, window script `/root/arms/usw-window.sh`, logs and `res.json` per arm in
|
||||
`/root/arms/usw-ab/{M1,U1,U2,M2}/`).** Production 35B think entry (`q36t`: `${ctx_256k}
|
||||
${qwen35_think_args} ${moe_offload_args}`, `-hf` pulls the mmproj, supervisord's environment, scratch
|
||||
arena and `--cache-disk-path`, plus `-lv 4`), one fresh server per arm, arms M (merged, default) and U
|
||||
(`LLAMA_USERSTART_MERGE=0`) in the order M U U M. Each chat: system prompt, 6 user turns from
|
||||
`pt-chunk.txt` (150, 10, 40, 150, 600, 40 tokens), synthetic previous answers of 300, 800, 1200, 500,
|
||||
1300 tokens, `max_tokens 8`, `temperature 0`, streamed; then the last user message edited in the
|
||||
middle and resent. TTFT in seconds (mean of the two runs per arm; the single runs in brackets),
|
||||
`prompt_n` was identical in all four runs on every turn:
|
||||
|
||||
| turn | user tokens | prompt_n | M TTFT | U TTFT | M - U |
|
||||
|---|---|---|---|---|---|
|
||||
| 1 (fresh prompt) | 150 | 196 | 3.99 (4.43, 3.55) | 6.08 (5.73, 6.42) | -2.1 |
|
||||
| 2 | 10 | 327 | 4.11 (3.75, 4.47) | 7.66 (6.75, 8.57) | -3.6 |
|
||||
| 3 | 40 | 869 | 6.34 (6.62, 6.06) | 9.34 (7.26, 11.42) | -3.0 |
|
||||
| 4 | 150 | 1381 | 7.20 (7.30, 7.10) | 10.95 (12.31, 9.58) | -3.7 |
|
||||
| 5 | 600 | 1130 | 8.82 (7.47, 10.17) | 9.80 (9.52, 10.07) | -1.0 |
|
||||
| 6 | 40 | 1370 | 8.11 (8.01, 8.20) | 9.53 (9.33, 9.72) | -1.4 |
|
||||
| edit of turn 6 | 40 | M 1374, U 66 | 7.94 (8.16, 7.71) | 3.65 (4.09, 3.20) | +4.3 |
|
||||
|
||||
- Batch layout (`prompt processing, n_tokens` and checkpoint lines at `-lv 4`): M runs every turn as
|
||||
one batch `[restore point, end-4)` plus the tail, with checkpoints at the batch start (superseding
|
||||
the restored one) and at end minus 4; U splits at the last user message (an extra checkpoint
|
||||
there, for example 1323 between 515 and 1380 on turn 3) and on turn 1 also at the first user
|
||||
message (system prompt, 23 tokens). Turn N+1 restores the end-minus-4 checkpoint of turn N in both
|
||||
arms. No `forcing full prompt re-processing` in any run.
|
||||
- The 10-token turn gained too, against the plan's "no change". U split it into 308 tokens (the
|
||||
previous answer, on the GPU) and 19 (message, generation prompt and tail), and the 19-token part runs
|
||||
the experts of all 40 layers on the CPU, which costs seconds rather than nothing at 6 threads
|
||||
(inferred from the totals, not profiled). The 600-token turn gained
|
||||
least (its two parts are 508 and 618 tokens, both on the GPU, so the split costs one extra
|
||||
upload, about 1-2 s against noise of the same size).
|
||||
- Edit of the last user message (the cost of the trade): M has no checkpoint at the user-message
|
||||
start, restores the batch start and re-processes previous answer + message (1374 tokens, one
|
||||
ubatch); U restores the user-start checkpoint and re-processes 66 tokens. +4.3 s per such edit.
|
||||
Ordinary follow-up turns are far more frequent than edits of the last message in this use; a client
|
||||
that injects per-request context into the last user message (RAG, web search) pays the same
|
||||
as an edit on every turn, so set `LLAMA_USERSTART_MERGE=0` for such a client.
|
||||
- Outputs: identical within each arm across runs; between arms turns 5 and 6 differ after a few
|
||||
tokens (batch-composition rounding, as the invariant says).
|
||||
- Disk cache: `fork-validate.sh --diskcache` (in the window, GPU leg included) 15 PASS, 0 FAIL, 0
|
||||
SKIP. `diskcache-window3.sh` legs with production's 35B command line (`q35.py` copied to
|
||||
`q35-usw.py` with the candidate build path; log `/root/arms/usw-q35-mm.log`): 18.8k-token chat,
|
||||
turn 1 83.5 s, save 557 MB in 1.0 s, restart restore and turn 2 `prompt_n` 27 in 3.7 s, output ==
|
||||
reference; cold save and restore identity (`prompt_n` 4, output == reference); image answered
|
||||
'Red' and not saved. The saved state after turn 2 holds 4 checkpoints instead of 5 (the user-start
|
||||
one is gone).
|
||||
|
||||
**Production (deployed 2026-11-22 10:37, `f3397cded`, config unchanged).** Deploy script
|
||||
`/root/arms/dep-usw.sh` (log `dep-usw.log`): incremental in-place build, downtime 33 s. Post-deploy
|
||||
checks through llama-swap (`/root/arms/post-usw.py`, `ink-usw.py`, `ink2-usw.py`, logs beside them):
|
||||
`system_fingerprint` `b12687-f3397cded`; 35B 'Paris', then 4B / 35B handoffs twice (correct answers,
|
||||
`holder` the answering PID each time, 2.9-6.3 s per switch); a 1140-token 35B chat, then a gemma-26B
|
||||
image ('Red', the 35B exited by itself with `loop stopped, device memory freed` and `disk cache:
|
||||
saved the prompt of 1157 tokens`), gemma text, 4B text, then the 35B chat plus one user message:
|
||||
`disk cache: restored 1157 tokens`, `prompt_n` 30 (the suffix only). Inkling nothink: text correct
|
||||
(it still spends about 40 tokens on reasoning, so a 96-token budget returns an empty content), image
|
||||
'Red', a 9293-token Pan Tadeusz prompt at 82.0 t/s with the right answer (F13 measured 81.6). Children
|
||||
bind llama-swap's ports. Log scan clean, `dmesg` clean (no ring timeout, GPU hang or OOM kill), pod
|
||||
restarts 0 as before.
|
||||
|
||||
**Failure modes and incidents.** None in validation or the deploy. Known cost: an edit of the last
|
||||
user message (or per-request context injected into it) re-processes the previous answer too, +4.3 s
|
||||
measured above.
|
||||
|
||||
**Rebase hazards.**
|
||||
- Upstream keeps reworking this loop (#20288, #22929, #24176, #25420): the user-start break, the
|
||||
`checkpoint_offsets` breaks and the near-end / min-step checkpoint rules. Re-check that the batch
|
||||
start is still checkpointed when the batch ends at prompt end minus 4.
|
||||
|
||||
**Validation plan (run 2026-11-22, results under Measured above).**
|
||||
1. Pre-check at trace log level (`SLT_TRC`; `-lv` high enough for `LOG_TRC`): per request, the
|
||||
`created context checkpoint ... n_tokens = N` and `restored context checkpoint` lines. Merged: one batch
|
||||
`[restore point, end-4)` plus `[end-4, end)`; `LLAMA_USERSTART_MERGE=0`: the extra break at the
|
||||
last user message.
|
||||
2. TTFT A/B through the production 35B entry (exact `moe_offload_args` command line, mmproj loaded),
|
||||
candidate with and without `LLAMA_USERSTART_MERGE=0`, arms alternated in one run: scripted 6-turn
|
||||
chats, user messages of about 10, 40, 150 and 600 tokens, previous answers 200-1500 tokens. Read
|
||||
`prompt_n`/`prompt_ms` from the timings and TTFT from the stream. Expected: no change at 10 tokens
|
||||
(the second part ran on the CPU), about -2 to -2.8 s per turn at 40+ tokens.
|
||||
3. Reuse: turn N+1 must report the same `prompt_n` in both arms (restore from end-minus-4). Then edit
|
||||
the last user message and resend: merged re-processes previous answer + message (bounded by one
|
||||
ubatch), unmerged only the message; no "forcing full prompt re-processing".
|
||||
4. Disk cache: `fork-validate.sh --diskcache` and `diskcache-window3.sh` (production command line):
|
||||
restore after a restart, `prompt_n` and output identical to the reference built from the same
|
||||
commit.
|
||||
|
||||
### F15. AVX2 NVFP4 x Q8_0 dot product (Inkling-2-Mini CPU experts)
|
||||
|
||||
Branch `fable5/nvfp4-avx2` (on the production branch tip `2caae7e11`). Not deployed. No knob: the
|
||||
old kernel is gone from the AVX2 path (drop = revert the commit).
|
||||
|
||||
**What and why.** Inkling-2-Mini UD-Q4_K_XL stores the experts of 20 of its 24 layers as NVFP4
|
||||
(ggml type 40: `block_nvfp4` = 4 UE4M3 scale bytes, one per 16 elements, then 32 bytes of E2M1
|
||||
nibbles; within each 8-byte sub-block group the low nibbles are elements 0-7 and the high nibbles
|
||||
8-15; values are the doubled E2M1 table `kvalues_fp4` and `ggml_ue4m3_to_fp32` returns the scale
|
||||
times 0.5; the per-expert `ffn_*_exps.scale` tensor is a separate graph multiply). The host experts
|
||||
(`-ot ...exps=CPU`) decode on the CPU through `ggml_vec_dot_nvfp4_q8_0`: the F13 profile had it at
|
||||
21% of decode CPU time, plus 36% in the OpenMP barrier. CPU_REPACK has no NVFP4 path, so vec_dot is
|
||||
what runs. Upstream's AVX2 loop did one block per iteration with 4 scalar UE4M3 table lookups, 2
|
||||
scalar FP16 conversions, 4 scalar muls, broadcasts and 2 FMAs on one accumulator.
|
||||
|
||||
**Where.** `ggml/src/ggml-cpu/arch/x86/quants.c`: `ue4m3x8_to_fp32_half` and the `__AVX2__` branch
|
||||
of `ggml_vec_dot_nvfp4_q8_0`. `tests/test-backend-ops.cpp`: perf cases at Inkling's expert shapes
|
||||
(nvfp4, mxfp4, q4_0; `m=2304`, k 2304 and 1152, n 1/8/16/32, MUL_MAT and MUL_MAT_ID 128x8). Commit
|
||||
`78a204bb2`.
|
||||
|
||||
**How it works.** Two blocks (128 weights) per iteration: one 32-byte load of `qs` per block, low and
|
||||
high nibbles through a 256-bit `pshufb` of `kvalues_fp4`, `unpack{lo,hi}_epi64` + `permute2x128`
|
||||
into the q8_0 element order, sign trick + `maddubs` against the two q8_0 blocks. The 8 sub-block sums
|
||||
are reduced exactly in int16 (at most 16*12*127 = 24384) by two levels of `hadd_epi16` and one
|
||||
`madd`, lanes `[b0s0 b0s2 b1s0 b1s2 | b0s1 b0s3 b1s1 b1s3]`. The 8 scale bytes are shuffled into the
|
||||
same lane order and decoded in one vector: normal `(x & 0x7F) << 20` plus exponent bias 119 (bias 7
|
||||
and the 0.5), subnormal `m * 2^-10` by blend, `0x7F` -> 0; bit-identical to `ggml_ue4m3_to_fp32` for
|
||||
all 256 bytes (including the sign-bit quirk: 0xFF decodes to 240, as in the reference). The 4 q8_0
|
||||
FP16 scales are packed in a GPR and converted with one `vcvtph2ps`. One mul + one FMA per
|
||||
iteration. An odd last block takes the existing scalar tail. The AVX-only path and the generic
|
||||
reference are untouched.
|
||||
|
||||
Tried and not kept (standalone harness `/root/arms/nvfp4-bench/nvfp4bench.c`, single core, min of
|
||||
400 trials): 4 blocks per iteration with two accumulators (no change), `srlv`-based nibble split with
|
||||
the q8_0 vector permuted instead (no change), `packs_epi32(madd)` instead of `hadd_epi16` (no
|
||||
change), the scale decode via a denormal multiply (`as_float(x << 20) * 2^119`, -4% time but wrong
|
||||
under DAZ, so not used). Ablation of the kept kernel: nibble decode + dot product about 60% of the
|
||||
time, the reduction 17%, the scales 23%. Zen 2 runs it at about 2.7 IPC with FP pipe 2 the busiest
|
||||
(`perf stat` `fpu_pipe_assignment`), not at the 4-pipe limit.
|
||||
|
||||
**Correctness.**
|
||||
- Standalone: bit-exact scale decode for all 256 bytes and the lane order; random blocks (wild and
|
||||
realistic scale bytes, n 64..2368, odd block counts) against the generic reference: worst error
|
||||
3.7e-8 of the sum of |terms|.
|
||||
- Linked against the built libraries (`vecdot_check.cpp`): 1400 rows of Gaussian weights at 4 scales
|
||||
(1e-3 drives the UE4M3 scales into subnormals), quantized with the library, vs the exact dot of the
|
||||
dequantized operands in double: new 5.5e-8, old 4.2e-8 (float rounding of the accumulation order).
|
||||
- `test-quantize-fns`: nvfp4 dot product error 0.019774 in both builds (identical).
|
||||
- `test-backend-ops -b CPU` cannot test a CPU kernel (it compares the CPU backend with itself).
|
||||
|
||||
**Measured (2026-11-22, pod, beside production, CPU-only scratch builds `-DGGML_VULKAN=OFF` in
|
||||
`/root/arms/nvfp4-base` (`2caae7e11`) and `/root/arms/nvfp4`; logs in `/root/arms/nvfp4-bench/`).**
|
||||
- Kernel, one pinned core, 64 rows L2-resident, min of 3000 trials, through the library's type
|
||||
traits (`vecdot_perf.cpp`, `vp1.log`, 4 runs per arm, spread 0.2 ns): k=2304 129.0 -> 113.8 ns/row
|
||||
(1.13x), k=1152 67.6 -> 59.5 (1.14x). Unchanged controls: q4_0 96.6 / 51.6, mxfp4 103.0 / 55.0 in
|
||||
both builds. NVFP4 is still about 18% slower per row than q4_0.
|
||||
- 6 threads streaming 512 MiB from DRAM (decode-like): every type, old or new, sits at the same
|
||||
21-23 GB/s (best of 15; medians 11-21 GB/s under the co-tenants). At 6 threads the expert dot
|
||||
product is DRAM-bound on this node, so the kernel speedup mostly turns into idle time, not decode
|
||||
speed. A decode token streams about 1.3 GB of host NVFP4 experts at production's k=4 (8 experts x
|
||||
36 layer passes x 35.8 MB).
|
||||
- `test-backend-ops perf` at 6 threads is useless here: the unchanged q4_0 and mxfp4 cases varied
|
||||
0.6x-1.7x between the two builds.
|
||||
- llama-bench CPU-only decode, `-ngl 0 -t 6 -p 0`, arms B N N B, 6 rounds: rounds 1-2 tg32 x3 (`lb.log`): base 5.23 5.12 5.04 5.19, new 5.52 5.34 5.23 5.15 t/s,
|
||||
medians 5.16 -> 5.29 (+2.5%); rounds 3-6 tg64 x5 (`lb2.log`): base 4.53 5.10 4.87 5.31 5.02 4.65 4.79
|
||||
4.93, new 4.98 5.24 5.34 5.09 4.70 4.38 5.02 5.18, medians 4.90 -> 5.06 (+3.2%). The new build won 5
|
||||
of the 6 rounds (mean of its two arms against the base pair). Below the 5% bar of section 6, so a
|
||||
likely +2-3%, not proven. CPU-only is the upper bound of the effect: every layer's experts and
|
||||
attention run on the CPU here; with production's k=4 the NVFP4 share of a token is smaller.
|
||||
|
||||
**What the GPU+CPU window should measure.** Production command line (`inkling_args`, k=4 `-ot`),
|
||||
llama-bench `-lm none -ngl 99 -ot <k=4 pattern> -fa 1 -t 6 -b 1024 -ub 1024 -p 0 -n 256 -d 4096 -r 3`
|
||||
(the F13 sweep's tg256 @ d4096 cell), base `2caae7e11` vs this branch, B N N B, medians (section 6).
|
||||
Expected: decode +0-3% (DRAM-bound), prefill unchanged (prefill MUL_MAT_ID runs on the GPU for
|
||||
ubatches >= 32). Optionally `perf record` the decode: `ggml_vec_dot_nvfp4_q8_0` should drop from 21%
|
||||
by roughly a tenth and the barrier share rise. Then a greedy text request against the base build
|
||||
(temperature 0): output may differ only by float rounding.
|
||||
|
||||
**How to drop it.** Revert `78a204bb2` (the kernel and the perf cases are independent hunks). Drop it
|
||||
when upstream rewrites this AVX2 path.
|
||||
|
||||
**Rebase hazards.** Upstream changes to `block_nvfp4` (layout, scale semantics, the 0.5 convention in
|
||||
`ggml_ue4m3_to_fp32`, the `0x7F`/sign handling TODO there) must be mirrored in
|
||||
`ue4m3x8_to_fp32_half`; the standalone check (`nvfp4bench.c`) tests all 256 bytes against the
|
||||
reference.
|
||||
|
||||
---
|
||||
|
||||
## 4. Dead ends
|
||||
@@ -1670,7 +1126,7 @@ layers.
|
||||
## 5. Configuration that matters
|
||||
|
||||
`/root/config.yaml` (llama-swap) and `/root/supervisord.conf` on the pod are the source of truth;
|
||||
this section explains the choices. Values as of the 2026-11-22 deploy.
|
||||
this section explains the choices. Values as of the 20261024 deploy.
|
||||
|
||||
Qwen 35B-A3B MoE entries (macro `moe_offload_args`, applied to all 35B-A3B variants):
|
||||
`--no-warmup --port ${PORT} -lm none -tb 12 -t 6 -ngl 99 --n-cpu-moe 40 -b 2048 -ub 2048 -fa 1 --parallel 1`,
|
||||
@@ -1705,27 +1161,6 @@ Other models:
|
||||
the task model killed it on every title request and forced full re-prefills. `--parallel 1` frees
|
||||
223 MiB of VRAM at 64k context there (4 auto slots size the SWA cache `n_swa * n_seq_max +
|
||||
n_ubatch`, 4096 cells instead of 2560).
|
||||
- Inkling-2-Mini (26B MoE, `inkling2`, 24 layers with the middle 12 looped 3x, 24 heads / 8 KV
|
||||
heads, head size 128; macro `inkling_args`, entries `unsloth/Inkling-2-Mini-GGUF[-nothink]:UD-Q4_K_XL`,
|
||||
matrix vars `ink`/`inknt`): the mainline-layout GGUF in `/root/models/inkling/` passed with `-m`
|
||||
(the repo-root GGUFs that `-hf` would pick are Unsloth's layout), `--mmproj` beside it,
|
||||
`-t 6`, f16 KV, `--parallel 1`, the disk prompt cache, `-b 1024 -ub 1024 --ctx-size 65536`
|
||||
(since the 2026-11-22 carry of upstream #32740, F12; before: `-b 512 -ub 512`). Since the F13
|
||||
deploy (2026-11-22, `b13c41355`): the experts of loop layers 6-9 stay on the GPU and all others
|
||||
on the host, `-ot blk[.](0|1|2|3|4|5|10|11|12|13|14|15|16|17|18|19|20|21|22|23)[.]ffn_.*_exps=CPU`
|
||||
instead of `--n-cpu-moe 24` (k=4 of the F13 placement guide: 7246 MiB of device memory, 868 MiB
|
||||
left for the mmproj). The `GGML_OP_OFFLOAD_MIN_BATCH=64` env of both entries was removed: F13
|
||||
makes the 32-63-token ubatches correct (KLD in F13). Other choices from the guide: ctx 32768 with
|
||||
k=6, or ctx 131072 with k=0 (fits since F13, 134 MiB free, so images would spill). Smaller quants
|
||||
(UD-Q3_K_XL, UD-IQ3_XXS, `/root/models/inkling/`) were measured and not adopted (F13, Quants).
|
||||
Its nothink macro uses `--reasoning-effort none` (no `--reasoning-budget 0`: that budget made it
|
||||
loop on chat-structure tokens, F12); even with effort none it writes a short reasoning block.
|
||||
Against Qwen3.6-35B (think entries, 7 tasks, one sample each, 3500-token cap, 2026-11-22,
|
||||
`/root/arms/overnight/quality.py`, `quality/quality.json`): Inkling 6/7, Qwen 5/7. Inkling found
|
||||
all three bugs of a binary search where Qwen ran out of budget; both ran out on a 4-line Polish
|
||||
rhyme. Inkling used about a third fewer tokens (9677 vs 14254) but took 1.7x the wall time (2283
|
||||
vs 1316 s; decode 4-6 vs 13 t/s) and 4x on a 29K-token question (835 vs 211 s). Too small a
|
||||
sample to call it smarter; it is clearly slower here.
|
||||
|
||||
Environment, set once for all of llama-swap in `[program:llama-swap] environment=` of
|
||||
`/root/supervisord.conf`:
|
||||
@@ -1739,7 +1174,7 @@ Environment, set once for all of llama-swap in `[program:llama-swap] environment
|
||||
|
||||
Not set, on purpose: `GGML_VK_DISABLE_GRAPH_OPTIMIZE`, `GGML_SCHED_PREFETCH_EXPERTS`,
|
||||
`GGML_VK_ASYNC_USE_TRANSFER_QUEUE`, `GGML_VK_CONCAT_FASTPATH`, `--moe-cache-mib`.
|
||||
`--cache-disk-path /root/prompt-cache --cache-disk-size 4096` (F11) is set in the `moe_offload_args`, `moe_offload_gemma` and `inkling_args` macros.
|
||||
`--cache-disk-path` (F11) is not set yet: it is not deployed; the recommended flags are in F11.
|
||||
|
||||
Machine-level:
|
||||
- GPU power cap fixed at 120 W by `/root/fan-control.sh` (a separate supervisord program; re-asserted
|
||||
@@ -1800,13 +1235,6 @@ Machine-level:
|
||||
`device_resident()` stays false, decode retries the restore on every call, and a graph reading a
|
||||
null buffer would crash. Only the mmproj path handles a failed restore.
|
||||
- Memory types without a `release_device_buffers` override keep their KV under `evict_kv` (F4).
|
||||
- Inkling (F12, section 5): the nothink entry's chat-structure loop came from
|
||||
`--reasoning-budget 0` and was fixed in the config on 2026-11-22 (F12). Upstream #32755 / #32778
|
||||
did not reproduce here; F13 bounds the loop-pass expert copies (more in F13). Drop the F12 carries when upstream merges #32735 and #32740.
|
||||
- Open after the 2026-11-21 rebase: is the fork's concat fast path (F3) now redundant with upstream's
|
||||
own tiled concat (#30591, gated in the fork with `ne0 < 32` and the `GGML_VK_CONCAT_FASTPATH` knob)?
|
||||
Benchmark it. Does `n_inflight` blocking the want-lock yield delay handoffs? Validation says no
|
||||
(handoff median 692 ms candidate vs 1068 ms reference).
|
||||
- Router mode can spin at shutdown (F7); not used here.
|
||||
- The unexplained ~1.6 GiB peak-VRAM spike in the first candidate `--perf` arm, seen twice (F9).
|
||||
- Not reported upstream: the `wait_until_no_sleep()` lost wakeup and the sleep wait that ignored new
|
||||
@@ -1814,10 +1242,10 @@ Machine-level:
|
||||
Repository rules forbid agent-written upstream issues or PRs; the owner would have to write them.
|
||||
- F10's load-time cost on the 35B (synchronous page freeing, about 0.28 s per GiB on the test model)
|
||||
is an estimate; its memory effect is verified in production.
|
||||
- F11 (disk prompt cache): the same-reuse-point identity check and the production-command-line (mmproj
|
||||
loaded) 35B run both passed on 2026-11-21 (`/root/arms/diskcache-window3.sh`, F11 Validation), and
|
||||
the cache is on in production. Remaining gap: only the 35B and gemma were checked in production;
|
||||
other models gain the flag only if they use those macros.
|
||||
- F11 (disk prompt cache): the same-reuse-point identity check (F11, Validation) has not run on the
|
||||
35B, only on the 0.8B (CPU and GPU), and no 35B run has used production's command line (mmproj
|
||||
loaded) yet. `/root/arms/diskcache-window3.sh` covers both (about 8 minutes of downtime). Do not
|
||||
deploy `--cache-disk-path` before it passes.
|
||||
- A SIGTERM during an arbiter release with `LLAMA_SLEEP_EVICT_KV` (F4, not F11) pays that release's
|
||||
host shadow of the full KV allocation (about 2.6 GB for the 35B at ctx 131072) and its D2H time
|
||||
(token free in about 1.9 s) before the exit. The same is true of every cold 35B, which holds that
|
||||
|
||||
@@ -1,28 +1,22 @@
|
||||
# llama.cpp
|
||||
|
||||

|
||||

|
||||
|
||||
<div align="center">
|
||||
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v0)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[v1.0.0](https://huggingface.co/blog/ggml-org/llama-cpp-v1) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal setup and state-of-the-art performance on a wide range of hardware - locally and in the cloud.
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## This fork - RX 580 (Polaris / GCN) serving of large MoE models
|
||||
|
||||
A private fork maintained for one machine: an AMD RX 580 8 GB (RADV Vulkan) serving Qwen3.6-35B-A3B,
|
||||
@@ -83,12 +77,50 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
<i>VLM session with <b>llama cli</b></i>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img width="1392" height="958" alt="Llama Chat, the built-in chat app, against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Llama Chat, the built-in chat app, against <b>llama serve</b></i>
|
||||
<img width="1392" height="958" alt="Built-in web UI against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Built-in web UI against <b>llama serve</b></i>
|
||||
</td>
|
||||
</tr>
|
||||
<table>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on
|
||||
a wide range of hardware - locally and in the cloud.
|
||||
|
||||
- Plain C/C++ implementation without any dependencies
|
||||
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
||||
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
||||
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
||||
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
||||
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
||||
- Vulkan and SYCL backend support
|
||||
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
||||
|
||||
The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Supported backends
|
||||
|
||||
| Backend | Target devices |
|
||||
| --- | --- |
|
||||
| [BLAS](docs/build.md#blas-build) | All |
|
||||
| [BLIS](docs/backend/BLIS.md) | All |
|
||||
| [CANN](docs/build.md#cann) | Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
||||
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
||||
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
||||
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
||||
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
||||
| [Vulkan](docs/build.md#vulkan) | GPU |
|
||||
| [WebGPU](docs/build.md#webgpu) | All |
|
||||
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
||||
|
||||
## Documentation
|
||||
|
||||
#### Tools
|
||||
@@ -96,7 +128,6 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
- [cli](tools/cli/README.md)
|
||||
- [completion](tools/completion/README.md)
|
||||
- [server](tools/server/README.md)
|
||||
- [Text-to-speech (TTS) models](docs/multimodal/tts.md)
|
||||
- [GBNF grammars](grammars/README.md)
|
||||
|
||||
#### Development
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ Only vulnerabilities that fall within these parts of the project are considered
|
||||
- `ggml/**/*`
|
||||
- `gguf-py/**/*`
|
||||
- `tools/server/*`, **excluding** the following topics:
|
||||
- Llama Chat, the built-in chat app (`ui/apps/chat`)
|
||||
- Web UI
|
||||
- Features marked as experimental
|
||||
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
||||
- Bugs that can lead to Denial-of-Service attack
|
||||
|
||||
@@ -122,8 +122,6 @@ if [ ! -z ${GG_BUILD_VULKAN} ]; then
|
||||
MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION="${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers/SPIRV-HeadersConfig.cmake"
|
||||
if [[ -f "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" || -h "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" ]]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DSPIRV-Headers_DIR=${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers"
|
||||
# SPIRV-Headers walks up three directories and appends /include, yielding /usr/local/lib/include; the extra vulkan/ level likely breaks its prefix calculation.
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN_SPIRV_INCLUDE_DIR=/usr/local/include"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -182,7 +180,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
|
||||
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXTRA="-E test-llama-archs|test-llama-context|test-fp8-kv"
|
||||
CTEST_EXTRA="-E test-llama-archs|test-llama-context"
|
||||
fi
|
||||
|
||||
## helpers
|
||||
@@ -298,6 +296,25 @@ function gg_run_test_llama_archs_tensor_split {
|
||||
set +e
|
||||
}
|
||||
|
||||
# test_llama_archs_models
|
||||
|
||||
function gg_run_test_llama_archs_models {
|
||||
cd ${SRC}
|
||||
|
||||
set -e
|
||||
|
||||
# TODO: fix and re-enable `test-llama-archs` on OpenVINO
|
||||
# TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
|
||||
if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
|
||||
# generate the dummy models used by the model-dependent tests
|
||||
./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
|
||||
fi
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
# test_scripts
|
||||
|
||||
function gg_run_test_scripts {
|
||||
@@ -439,10 +456,14 @@ function gg_run_qwen3_0_6b {
|
||||
|
||||
(time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
|
||||
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
# TODO: these tests are extremely slow on WebGPU - re-enable when fixed
|
||||
# https://github.com/ggml-org/llama.cpp/actions/runs/39639985732
|
||||
if [ -z ${GG_BUILD_WEBGPU} ]; then
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
fi
|
||||
|
||||
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")"
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
@@ -680,6 +701,7 @@ test $ret -eq 0 && gg_run ctest_release
|
||||
|
||||
test $ret -eq 0 && gg_run test_backend_ops
|
||||
|
||||
test $ret -eq 0 && gg_run test_llama_archs_models
|
||||
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
|
||||
|
||||
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
|
||||
@@ -2,11 +2,11 @@ get_filename_component(DEST_DIR "${DEST}" DIRECTORY)
|
||||
file(MAKE_DIRECTORY "${DEST_DIR}")
|
||||
|
||||
if(NOT EXISTS "${DEST}")
|
||||
message(STATUS "Downloading ${NAME} from ${REPO}...")
|
||||
message(STATUS "Downloading ${NAME} from ggml-org/models...")
|
||||
endif()
|
||||
|
||||
file(DOWNLOAD
|
||||
"https://huggingface.co/${REPO}/resolve/main/${NAME}?download=true"
|
||||
"https://huggingface.co/ggml-org/models/resolve/main/${NAME}?download=true"
|
||||
"${DEST}"
|
||||
TLS_VERIFY ON
|
||||
EXPECTED_HASH ${HASH}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
set(LLAMA_WIN_APP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/../common/win-app.manifest")
|
||||
|
||||
if(WIN32)
|
||||
if(MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
|
||||
set(LLAMA_WIN_APP_MSVC_ABI TRUE)
|
||||
else()
|
||||
set(LLAMA_WIN_APP_MSVC_ABI FALSE)
|
||||
endif()
|
||||
|
||||
if(NOT LLAMA_WIN_APP_MSVC_ABI)
|
||||
if(CMAKE_VERSION VERSION_LESS 3.21)
|
||||
message(FATAL_ERROR "CMake 3.21 or newer is required to build for Windows with the GNU ABI")
|
||||
endif()
|
||||
enable_language(RC)
|
||||
set(LLAMA_WIN_APP_MANIFEST_RC "${CMAKE_CURRENT_BINARY_DIR}/win-app-manifest.rc")
|
||||
file(CONFIGURE OUTPUT "${LLAMA_WIN_APP_MANIFEST_RC}" CONTENT "1 24 \"@LLAMA_WIN_APP_MANIFEST@\"\n" @ONLY)
|
||||
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
function(llama_add_app_manifest target)
|
||||
if(NOT WIN32)
|
||||
return()
|
||||
endif()
|
||||
get_target_property(_aliased ${target} ALIASED_TARGET)
|
||||
if(_aliased)
|
||||
return()
|
||||
endif()
|
||||
get_target_property(_type ${target} TYPE)
|
||||
if(NOT _type STREQUAL "EXECUTABLE")
|
||||
return()
|
||||
endif()
|
||||
if(LLAMA_WIN_APP_MSVC_ABI)
|
||||
target_sources(${target} PRIVATE "${LLAMA_WIN_APP_MANIFEST}")
|
||||
else()
|
||||
if(NOT TARGET llama-win-app-manifest-res)
|
||||
add_library(llama-win-app-manifest-res OBJECT ${LLAMA_WIN_APP_MANIFEST_RC})
|
||||
set_source_files_properties(${LLAMA_WIN_APP_MANIFEST_RC} PROPERTIES OBJECT_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
|
||||
endif()
|
||||
target_link_libraries(${target} PRIVATE "$<TARGET_OBJECTS:llama-win-app-manifest-res>")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(llama_apply_app_manifests_dir dir)
|
||||
get_property(_targets DIRECTORY "${dir}" PROPERTY BUILDSYSTEM_TARGETS)
|
||||
foreach(_target IN LISTS _targets)
|
||||
llama_add_app_manifest(${_target})
|
||||
endforeach()
|
||||
get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES)
|
||||
foreach(_subdir IN LISTS _subdirs)
|
||||
llama_apply_app_manifests_dir("${_subdir}")
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
function(llama_apply_app_manifests)
|
||||
if(NOT WIN32)
|
||||
return()
|
||||
endif()
|
||||
get_property(_subdirs DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY SUBDIRECTORIES)
|
||||
foreach(_dir IN LISTS _subdirs)
|
||||
get_filename_component(_name "${_dir}" NAME)
|
||||
if(_name STREQUAL "app" OR _name STREQUAL "examples" OR _name STREQUAL "pocs" OR _name STREQUAL "tests" OR _name STREQUAL "tools")
|
||||
llama_apply_app_manifests_dir("${_dir}")
|
||||
endif()
|
||||
endforeach()
|
||||
endfunction()
|
||||
@@ -66,8 +66,6 @@ add_library(${TARGET}
|
||||
chat-diff-analyzer.cpp
|
||||
chat-peg-parser.cpp
|
||||
chat-peg-parser.h
|
||||
chat-tts-norm.cpp
|
||||
chat-tts-norm.h
|
||||
chat.cpp
|
||||
chat.h
|
||||
common.cpp
|
||||
|
||||
+86
-116
@@ -4,7 +4,6 @@
|
||||
#include "chat.h"
|
||||
#include "common.h"
|
||||
#include "download.h"
|
||||
#include "http.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "json.h"
|
||||
#include "llama.h"
|
||||
@@ -13,6 +12,16 @@
|
||||
#include "speculative.h"
|
||||
#include "preset.h"
|
||||
|
||||
// fix problem with std::min and std::max
|
||||
#if defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#ifndef NOMINMAX
|
||||
# define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#include <shellapi.h>
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
#include <climits>
|
||||
@@ -29,6 +38,20 @@
|
||||
#include <thread> // for hardware_concurrency
|
||||
#include <vector>
|
||||
|
||||
#ifndef __EMSCRIPTEN__
|
||||
#ifdef __linux__
|
||||
#include <linux/limits.h>
|
||||
#elif defined(_WIN32)
|
||||
# if !defined(PATH_MAX)
|
||||
# define PATH_MAX MAX_PATH
|
||||
# endif
|
||||
#elif defined(_AIX)
|
||||
#include <sys/limits.h>
|
||||
#else
|
||||
#include <sys/syslimits.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
|
||||
|
||||
using json = common_json;
|
||||
@@ -282,7 +305,6 @@ const std::vector<ggml_type> kv_cache_types = {
|
||||
GGML_TYPE_F32,
|
||||
GGML_TYPE_F16,
|
||||
GGML_TYPE_BF16,
|
||||
GGML_TYPE_F8_E4M3,
|
||||
GGML_TYPE_Q8_0,
|
||||
GGML_TYPE_Q4_0,
|
||||
GGML_TYPE_Q4_1,
|
||||
@@ -329,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
|
||||
static std::string get_default_local_path(const std::string & url) {
|
||||
auto f = string_split<std::string>(url, '#').front();
|
||||
f = string_split<std::string>(f, '?').front();
|
||||
return fs_get_cache_file(string_split<std::string>(f, '/').back()).string();
|
||||
return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
|
||||
}
|
||||
|
||||
static bool spec_types_is_default(const common_params & params) {
|
||||
@@ -349,14 +371,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_dflash2 = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_lilicorr = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_eagle3 = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();
|
||||
@@ -381,7 +395,7 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
opts.offline = params.offline;
|
||||
opts.download_mtp = spec_type_draft_mtp;
|
||||
opts.download_eagle3 = spec_type_draft_eagle3;
|
||||
opts.download_dflash = spec_type_draft_dflash || spec_type_draft_dflash2 || spec_type_draft_lilicorr;
|
||||
opts.download_dflash = spec_type_draft_dflash;
|
||||
opts.download_dspark = spec_type_draft_dspark;
|
||||
opts.download_mmproj = use_mmproj && !params.no_mmproj
|
||||
&& params.mmproj.path.empty() && params.mmproj.url.empty();
|
||||
@@ -409,26 +423,7 @@ bool common_models_handler_is_preset_repo(const common_models_handler & handler)
|
||||
return !handler.plan.preset.url.empty();
|
||||
}
|
||||
|
||||
static common_download_opts download_opts_for_url(const common_download_opts & opts, const std::string & url) {
|
||||
common_download_opts result = opts;
|
||||
try {
|
||||
const auto hf = common_http_parse_url(common_get_model_endpoint());
|
||||
const auto parts = common_http_parse_url(url);
|
||||
|
||||
if (parts.scheme == hf.scheme &&
|
||||
parts.host == hf.host &&
|
||||
parts.port == hf.port &&
|
||||
string_starts_with(result.bearer_token, "hf_")) {
|
||||
return result;
|
||||
}
|
||||
} catch (const std::exception &) {
|
||||
}
|
||||
result.bearer_token.clear();
|
||||
return result;
|
||||
}
|
||||
|
||||
static std::vector<common_download_task> build_url_tasks(const common_params_model & model, common_download_opts opts) {
|
||||
opts = download_opts_for_url(opts, model.url);
|
||||
auto parts = common_download_get_all_parts(model.url);
|
||||
std::vector<common_download_task> tasks;
|
||||
|
||||
@@ -517,7 +512,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
common_download_task task;
|
||||
task.url = params.mmproj.url;
|
||||
task.local_path = params.mmproj.path;
|
||||
task.opts = download_opts_for_url(opts, task.url);
|
||||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
}
|
||||
bool had_spec_url = false;
|
||||
@@ -525,7 +520,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
common_download_task task;
|
||||
task.url = params.speculative.draft.mparams.url;
|
||||
task.local_path = params.speculative.draft.mparams.path;
|
||||
task.opts = download_opts_for_url(opts, task.url);
|
||||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
had_spec_url = true;
|
||||
}
|
||||
@@ -739,7 +734,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
|
||||
std::vector<std::filesystem::path> paths;
|
||||
|
||||
#if defined(_WIN32)
|
||||
const std::filesystem::path program_data = common_get_env("PROGRAMDATA");
|
||||
const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
|
||||
if (!program_data.empty()) {
|
||||
paths.push_back(program_data / "llama.cpp" / "config.ini");
|
||||
}
|
||||
@@ -767,7 +762,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
|
||||
common_preset_context ctx(ex);
|
||||
ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
|
||||
for (const auto & path : found) {
|
||||
LOG_INF("using config file: %s\n", path.string().c_str());
|
||||
LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
|
||||
common_preset global;
|
||||
common_presets presets = ctx.load_from_ini(path, global);
|
||||
global.apply_to_params(params);
|
||||
@@ -912,10 +907,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
|
||||
}
|
||||
|
||||
if (params.sampling.ignore_eos && !common_grammar_value(params.sampling.grammar).empty()) {
|
||||
throw std::invalid_argument("error: --ignore-eos is not supported with --grammar or --json-schema\n");
|
||||
}
|
||||
|
||||
const bool skip_model_download =
|
||||
// server will call common_params_handle_models() later, so we skip it here
|
||||
ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||
|
||||
@@ -933,7 +924,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
// TODO @ngxson : maybe show a list of available models in CLI in this case
|
||||
bool can_skip_model = params.usage || params.completion || !params.server_base.empty();
|
||||
if (!can_skip_model && params.model.path.empty()) {
|
||||
throw std::invalid_argument("error: --model or -hf is required\n");
|
||||
throw std::invalid_argument("error: --model is required\n");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -955,8 +946,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
}
|
||||
|
||||
const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty();
|
||||
if ((!params.server_tools.empty() || mcp_enabled || params.ui_mcp_proxy) && !params.cors_origins_explicit) {
|
||||
LOG_WRN("server tools, MCP servers or MCP proxy are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
|
||||
if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) {
|
||||
LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
|
||||
params.cors_origins = "localhost";
|
||||
}
|
||||
|
||||
@@ -1268,7 +1259,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
|
||||
return true;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
struct utf8_argv {
|
||||
std::vector<std::string> buf;
|
||||
std::vector<char*> ptrs;
|
||||
};
|
||||
|
||||
static utf8_argv make_utf8_argv() {
|
||||
utf8_argv out;
|
||||
int wargc = 0;
|
||||
LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
|
||||
if (!wargv) return out;
|
||||
|
||||
out.buf.reserve(wargc);
|
||||
for (int i = 0; i < wargc; ++i) {
|
||||
int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);
|
||||
if (n <= 0) { out.buf.emplace_back(); continue; }
|
||||
auto& s = out.buf.emplace_back();
|
||||
s.resize(static_cast<size_t>(n - 1));
|
||||
(void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);
|
||||
}
|
||||
LocalFree(wargv);
|
||||
|
||||
out.ptrs.reserve(out.buf.size() + 1);
|
||||
for (auto& s : out.buf) out.ptrs.push_back(s.data());
|
||||
out.ptrs.push_back(nullptr);
|
||||
return out;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
|
||||
#ifdef _WIN32
|
||||
auto utf8 = make_utf8_argv();
|
||||
// repair argv only when it matches the process command line
|
||||
if (static_cast<int>(utf8.buf.size()) == argc) {
|
||||
argv = utf8.ptrs.data();
|
||||
}
|
||||
#endif
|
||||
|
||||
auto ctx_arg = common_params_parser_init(params, ex, print_usage);
|
||||
const common_params params_org = ctx_arg.params; // the example can modify the default params
|
||||
|
||||
@@ -2530,6 +2558,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.ppl_output_type = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
|
||||
add_opt(common_arg(
|
||||
{"-dt", "--defrag-thold"}, "N",
|
||||
string_format("KV cache defragmentation threshold (DEPRECATED)"),
|
||||
[](common_params & params, const std::string & value) {
|
||||
GGML_UNUSED(params);
|
||||
GGML_UNUSED(value);
|
||||
LOG_WRN("DEPRECATED: --defrag-thold is deprecated and no longer necessary to specify\n");
|
||||
}
|
||||
).set_env("LLAMA_ARG_DEFRAG_THOLD"));
|
||||
if (ex == LLAMA_EXAMPLE_SERVER) {
|
||||
// this is to make sure this option appears in the server-specific section of the help message
|
||||
add_opt(common_arg(
|
||||
@@ -2775,17 +2812,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.moe_cache_size = (size_t) value*1024*1024;
|
||||
}
|
||||
).set_env("LLAMA_ARG_MOE_CACHE_MIB"));
|
||||
add_opt(common_arg(
|
||||
{"--moe-cache-cpu-share"}, "F",
|
||||
string_format("share of the MoE cache misses that the CPU computes from the host experts instead of uploading them to the GPU once the cache is full, from 0 to 1 (default: %.2f)", (double) params.moe_cache_cpu_share),
|
||||
[](common_params & params, const std::string & value) {
|
||||
const float share = std::stof(value);
|
||||
if (share < 0.0f || share > 1.0f) {
|
||||
throw std::invalid_argument("invalid value");
|
||||
}
|
||||
params.moe_cache_cpu_share = share;
|
||||
}
|
||||
).set_env("LLAMA_ARG_MOE_CACHE_CPU_SHARE"));
|
||||
add_opt(common_arg(
|
||||
{"-ncffn", "--n-cpu-ffn"}, "N",
|
||||
"keep the dense FFN weights of the first N layers in the CPU\n"
|
||||
@@ -2850,7 +2876,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -2916,7 +2942,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -3053,16 +3079,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.model.path = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
|
||||
add_opt(common_arg(
|
||||
{"--ple", "--shard"}, "FNAME,...",
|
||||
"extra GGUF shards for the model, comma separated; may be given more than once\n"
|
||||
"paths may live anywhere, including on different drives; --ple is an alias for --shard",
|
||||
[](common_params & params, const std::string & value) {
|
||||
for (const auto & item : parse_csv_row(value)) {
|
||||
params.model.shards.push_back(item);
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD"));
|
||||
add_opt(common_arg(
|
||||
{"-mu", "--model-url"}, "MODEL_URL",
|
||||
"model download url (default: unused)",
|
||||
@@ -3381,7 +3397,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
{"--cors-origins"}, "ORIGINS",
|
||||
string_format(
|
||||
"comma-separated list of allowed origins for CORS (default: %s)\n"
|
||||
"if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)",
|
||||
"if set to special value 'localhost', reflect the Origin header only if it is localhost",
|
||||
params.cors_origins.c_str()),
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.cors_origins = value;
|
||||
@@ -3437,20 +3453,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused unless --ui-mcp-proxy-allow-private-urls is set - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
"experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY"));
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy-allow-private-urls"},
|
||||
"allow the MCP CORS proxy to reach local or private addresses (e.g. MCP servers on localhost or LAN) - anyone who can reach the proxy can then reach your local network (default: disabled, only public addresses)\n"
|
||||
"note: only has effect with --ui-mcp-proxy",
|
||||
[](common_params & params) {
|
||||
params.ui_mcp_proxy_allow_private_urls = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY_ALLOW_PRIVATE_URLS"));
|
||||
add_opt(common_arg(
|
||||
{"--tools"}, "TOOL1,TOOL2,...",
|
||||
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
|
||||
@@ -3472,13 +3479,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.server_tools_runtime = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME"));
|
||||
add_opt(common_arg(
|
||||
{"--allow-unauthenticated-tools"},
|
||||
"allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)",
|
||||
[](common_params & params) {
|
||||
params.server_tools_allow_unauth = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS"));
|
||||
add_opt(common_arg(
|
||||
{"--mcp-servers-config"}, "PATH",
|
||||
"experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n"
|
||||
@@ -3514,7 +3514,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui", "--webui"},
|
||||
{"--no-ui", "--no-webui"},
|
||||
string_format("whether to enable Llama Chat, the built-in chat app (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
[](common_params & params, bool value) {
|
||||
params.ui = value;
|
||||
}
|
||||
@@ -3704,13 +3704,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--media-allow-private-urls"},
|
||||
"allow http(s) media URLs (e.g. image_url) that point to local or private addresses (default: disabled, only public addresses)",
|
||||
[](common_params & params) {
|
||||
params.media_allow_private_urls = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MEDIA_ALLOW_PRIVATE_URLS"));
|
||||
add_opt(common_arg(
|
||||
{"--models-dir"}, "PATH",
|
||||
"directory containing models for the router server (default: disabled)",
|
||||
@@ -3732,13 +3725,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.models_max = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MAX"));
|
||||
add_opt(common_arg(
|
||||
{"--models-memory-margin"}, "N",
|
||||
string_format("for router server, MiB of device memory to keep free per device, unload LRU models until a new one fits (default: %d, 0 = disabled)", params.models_memory_margin),
|
||||
[](common_params & params, int value) {
|
||||
params.models_memory_margin = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MEMORY_MARGIN"));
|
||||
add_opt(common_arg(
|
||||
{"--models-autoload"},
|
||||
{"--no-models-autoload"},
|
||||
@@ -4531,19 +4517,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--tts-lang"}, "FNAME",
|
||||
"language (ISO 639-1) for audio generation\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
"see tts/README.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_lang = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-voice"}, "VOICE",
|
||||
"voice for audio generation (model-dependent)\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_voice = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-file"}, "FNAME",
|
||||
"speaker file path for audio generation",
|
||||
@@ -4551,14 +4529,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.tts_speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-text"}, "TEXT",
|
||||
"transcript of --tts-speaker-file, for models that need it to clone the voice\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_text = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
//
|
||||
// diffusion params
|
||||
|
||||
+49
-67
@@ -131,12 +131,6 @@ static std::string normalize_quotes_to_json(const std::string & input) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Models can emit raw control characters inside string literals; escape them to keep the JSON valid.
|
||||
if ((in_single_quoted || in_double_quoted) && static_cast<unsigned char>(c) < 0x20) {
|
||||
result += escape_json_string_inner(std::string(1, c));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle quote characters
|
||||
if (c == '"') {
|
||||
if (in_single_quoted) {
|
||||
@@ -354,8 +348,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
|
||||
}
|
||||
|
||||
if (is_tool_id && current_tool) {
|
||||
auto sanitized = node.sanitized_text();
|
||||
auto text = trim_trailing_space(sanitized);
|
||||
auto text = trim_trailing_space(node.text);
|
||||
if (text.size() >= 2 && text.front() == '"' && text.back() == '"') {
|
||||
text = text.substr(1, text.size() - 2);
|
||||
}
|
||||
@@ -408,7 +401,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
|
||||
}
|
||||
|
||||
if ((is_arg_value || is_arg_string_value) && current_tool) {
|
||||
std::string value_content = node.sanitized_text();
|
||||
std::string value_content = std::string(node.text);
|
||||
|
||||
std::string value_to_add;
|
||||
if (value_content.empty() && is_arg_string_value) {
|
||||
@@ -562,8 +555,7 @@ common_peg_parser common_chat_peg_builder::python_or_json_value() {
|
||||
common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
const ordered_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared) {
|
||||
bool allow_json_literals) {
|
||||
if (!tools.is_array() || tools.empty()) {
|
||||
return eps();
|
||||
}
|
||||
@@ -621,18 +613,6 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
|
||||
}
|
||||
|
||||
// last, so declared tools win when parsing
|
||||
if (allow_undeclared) {
|
||||
auto ident = rule("tool-undeclared-ident", chars("[a-zA-Z_]", 1, 1) + chars("[a-zA-Z0-9_.]", 0, -1));
|
||||
auto value = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());
|
||||
auto arg = tool_arg(tool_arg_open(tool_arg_name(ident) + literal("=")) + value + tool_arg_close(eps()));
|
||||
auto args = optional(arg + zero_or_more("," + space() + arg));
|
||||
|
||||
tool_choices |= rule("tool-undeclared", tool(tool_open(tool_name(ident) + literal("(")) +
|
||||
space() + tool_args(args) + space() + tool_close(literal(")"))
|
||||
));
|
||||
}
|
||||
|
||||
if (parallel_tool_calls) {
|
||||
return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]";
|
||||
}
|
||||
@@ -986,21 +966,6 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena,
|
||||
}
|
||||
}
|
||||
|
||||
using gemma4 = common_chat_peg_gemma4_mapper;
|
||||
|
||||
// Collects the nearest descendants carrying tag, looking through the rules the parser
|
||||
// wraps around the tagged nodes.
|
||||
static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector<common_peg_ast_id> & out) {
|
||||
for (auto child_id : node.children) {
|
||||
const auto & child = arena.get(child_id);
|
||||
if (child.tag == tag) {
|
||||
out.push_back(child_id);
|
||||
} else {
|
||||
gemma4_collect(arena, child, tag, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {
|
||||
const auto & node = arena.get(id);
|
||||
|
||||
@@ -1008,45 +973,51 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
|
||||
return "";
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) {
|
||||
if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {
|
||||
return std::string(node.text);
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::STRING_CONTENT) {
|
||||
return escape_json_string_inner(node.sanitized_text());
|
||||
if (node.rule == "gemma4-string-content") {
|
||||
return escape_json_string_inner(std::string(node.text));
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::STRING) {
|
||||
if (node.rule == "gemma4-string") {
|
||||
std::string result = "\"";
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
if (!node.is_partial) {
|
||||
result += "\"";
|
||||
if (!node.children.empty()) {
|
||||
result += gemma4_to_json(arena, node.children[0]);
|
||||
if (!node.is_partial) {
|
||||
result += "\"";
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::ARRAY) {
|
||||
if (node.rule == "gemma4-array") {
|
||||
std::string result = "[";
|
||||
for (size_t i = 0; i < node.children.size(); i++) {
|
||||
if (i > 0) {
|
||||
|
||||
bool add_comma = false;
|
||||
for (auto child_id : node.children) {
|
||||
if (add_comma) {
|
||||
result += ',';
|
||||
}
|
||||
result += gemma4_to_json(arena, node.children[i]);
|
||||
add_comma = true;
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
|
||||
if (!node.is_partial) {
|
||||
result += ']';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::DICT_KEY) {
|
||||
auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME);
|
||||
if (node.rule == "gemma4-dict-key-name") {
|
||||
return std::string(node.text);
|
||||
}
|
||||
|
||||
if (node.rule == "gemma4-dict-key") {
|
||||
std::string result = "\"";
|
||||
if (name_id != COMMON_PEG_INVALID_AST_ID) {
|
||||
result += escape_json_string_inner(std::string(arena.get(name_id).text));
|
||||
if (!node.children.empty()) {
|
||||
result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));
|
||||
}
|
||||
if (!node.is_partial) {
|
||||
result += "\":";
|
||||
@@ -1054,29 +1025,40 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::DICT) {
|
||||
std::vector<common_peg_ast_id> members;
|
||||
gemma4_collect(arena, node, gemma4::DICT_KV, members);
|
||||
if (node.rule == "gemma4-dict-kv") {
|
||||
std::string result;
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.rule == "gemma4-dict") {
|
||||
std::string result = "{";
|
||||
for (size_t i = 0; i < members.size(); i++) {
|
||||
if (i > 0) {
|
||||
|
||||
bool add_comma = false;
|
||||
for (auto child_id : node.children) {
|
||||
if (add_comma) {
|
||||
result += ',';
|
||||
}
|
||||
result += gemma4_to_json(arena, members[i]);
|
||||
add_comma = true;
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
|
||||
if (!node.is_partial) {
|
||||
result += '}';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// members, rules and other wrappers
|
||||
std::string result;
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
if (node.rule == "gemma4-value") {
|
||||
if (!node.children.empty()) {
|
||||
return gemma4_to_json(arena, node.children[0]);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
return result;
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {
|
||||
@@ -1213,7 +1195,7 @@ static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena,
|
||||
}
|
||||
|
||||
if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) {
|
||||
return "\"" + escape_json_string_inner(node.sanitized_text()) + (closed ? "\"" : "");
|
||||
return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : "");
|
||||
}
|
||||
|
||||
// Numbers and booleans are written verbatim by the template
|
||||
|
||||
@@ -34,18 +34,6 @@ class common_chat_peg_mapper {
|
||||
|
||||
class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {
|
||||
public:
|
||||
// Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules.
|
||||
static constexpr const char * STRING = "gemma4-string";
|
||||
static constexpr const char * STRING_CONTENT = "gemma4-string-content";
|
||||
static constexpr const char * NUMBER = "gemma4-number";
|
||||
static constexpr const char * BOOL = "gemma4-bool";
|
||||
static constexpr const char * NULL_ = "gemma4-null";
|
||||
static constexpr const char * ARRAY = "gemma4-array";
|
||||
static constexpr const char * DICT = "gemma4-dict";
|
||||
static constexpr const char * DICT_KV = "gemma4-dict-kv";
|
||||
static constexpr const char * DICT_KEY = "gemma4-dict-key";
|
||||
static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name";
|
||||
|
||||
common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}
|
||||
virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);
|
||||
private:
|
||||
@@ -161,11 +149,9 @@ class common_chat_peg_builder : public common_peg_parser_builder {
|
||||
|
||||
// Helper for Python-style function call format: name(arg1="value1", arg2=123)
|
||||
// Used by LFM2 and similar templates
|
||||
// allow_undeclared: also accept calls to names not in tools (any identifier, any arguments)
|
||||
common_peg_parser python_style_tool_calls(const common_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared = false);
|
||||
bool allow_json_literals);
|
||||
|
||||
private:
|
||||
// Python values plus JSON true/false/null.
|
||||
|
||||
@@ -1,902 +0,0 @@
|
||||
#include "chat-tts-norm.h"
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
|
||||
// keep this file in sync with the python reference: scripts/chat_tts_norm.py
|
||||
|
||||
using namespace jinja;
|
||||
|
||||
//
|
||||
// character classes
|
||||
// they are defined with explicit codepoint ranges, independent of any unicode database version,
|
||||
// so that the C++ and python implementations agree on every codepoint
|
||||
//
|
||||
|
||||
struct tts_norm_range {
|
||||
uint32_t lo;
|
||||
uint32_t hi;
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_SPACE[] = {
|
||||
{0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
|
||||
{0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_PUNCT[] = {
|
||||
{0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
|
||||
{0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
|
||||
{0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
|
||||
{0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
|
||||
{0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
|
||||
{0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
|
||||
{0x1F000, 0x1FAFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_CONTROL[] = {
|
||||
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_HAN[] = {
|
||||
{0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
|
||||
{0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
|
||||
};
|
||||
|
||||
template<size_t N>
|
||||
static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
|
||||
for (const auto & r : ranges) {
|
||||
if (c >= r.lo && c <= r.hi) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_alpha(uint32_t c) {
|
||||
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
|
||||
return true;
|
||||
}
|
||||
if (c == 0xAA || c == 0xB5 || c == 0xBA) {
|
||||
return true;
|
||||
}
|
||||
return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
|
||||
// kind is one of "dspawulhc"
|
||||
static bool tts_norm_class_match(char kind, uint32_t c) {
|
||||
switch (kind) {
|
||||
case 'd': return c >= '0' && c <= '9';
|
||||
case 's': return tts_norm_in(TTS_NORM_SPACE, c);
|
||||
case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
|
||||
case 'a': return tts_norm_is_alpha(c);
|
||||
case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
|
||||
case 'u': return c >= 'A' && c <= 'Z';
|
||||
case 'l': return c >= 'a' && c <= 'z';
|
||||
case 'h': return tts_norm_in(TTS_NORM_HAN, c);
|
||||
case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_class_letter(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') {
|
||||
c += 'a' - 'A';
|
||||
}
|
||||
return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_lower(uint32_t c) {
|
||||
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
|
||||
if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
|
||||
return c;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_ascii_alnum(uint32_t c) {
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
|
||||
// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
|
||||
static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
|
||||
cpts.clear();
|
||||
offs.clear();
|
||||
size_t i = 0;
|
||||
const size_t n = s.size();
|
||||
while (i < n) {
|
||||
offs.push_back(i);
|
||||
const unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
uint32_t cp = 0;
|
||||
size_t len = 0;
|
||||
if (c < 0x80) {
|
||||
cp = c; len = 1;
|
||||
} else if ((c & 0xE0) == 0xC0) {
|
||||
cp = c & 0x1F; len = 2;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
cp = c & 0x0F; len = 3;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
cp = c & 0x07; len = 4;
|
||||
}
|
||||
bool ok = len > 0 && i + len <= n;
|
||||
for (size_t k = 1; ok && k < len; k++) {
|
||||
const unsigned char cc = static_cast<unsigned char>(s[i + k]);
|
||||
if ((cc & 0xC0) != 0x80) {
|
||||
ok = false;
|
||||
} else {
|
||||
cp = (cp << 6) | (cc & 0x3F);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
cpts.push_back(0xFFFD);
|
||||
i += 1;
|
||||
} else {
|
||||
cpts.push_back(cp);
|
||||
i += len;
|
||||
}
|
||||
}
|
||||
offs.push_back(n);
|
||||
}
|
||||
|
||||
//
|
||||
// AST
|
||||
//
|
||||
|
||||
enum class tts_norm_op {
|
||||
LITERAL, // one codepoint
|
||||
SET, // one codepoint from a set; class escapes like %d are sets with a single class
|
||||
ANY, // any codepoint
|
||||
START, // ^
|
||||
END, // $
|
||||
BOUNDARY, // %b
|
||||
GROUP, // (...), (?:...), (?<name>...)
|
||||
LOOKAHEAD, // (?=...), (?!...)
|
||||
LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
|
||||
};
|
||||
|
||||
struct tts_norm_class {
|
||||
char kind;
|
||||
bool neg;
|
||||
};
|
||||
|
||||
struct tts_norm_set {
|
||||
bool neg = false;
|
||||
std::vector<std::pair<uint32_t, uint32_t>> ranges;
|
||||
std::vector<tts_norm_class> classes;
|
||||
};
|
||||
|
||||
struct tts_norm_item;
|
||||
using tts_norm_seq = std::vector<tts_norm_item>;
|
||||
|
||||
struct tts_norm_pattern_node {
|
||||
tts_norm_op op = tts_norm_op::GROUP;
|
||||
uint32_t cp = 0; // LITERAL
|
||||
tts_norm_set set; // SET
|
||||
std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
|
||||
int capture = -1; // GROUP: capture index, -1 for non-capturing
|
||||
bool neg = false; // LOOKAHEAD, LOOKBEHIND
|
||||
};
|
||||
|
||||
static constexpr int TTS_NORM_INF = INT_MAX;
|
||||
static constexpr int TTS_NORM_MAX_REPEAT = 1000;
|
||||
static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
|
||||
|
||||
struct tts_norm_item {
|
||||
tts_norm_pattern_node atom;
|
||||
int min = 1;
|
||||
int max = 1;
|
||||
};
|
||||
|
||||
static bool tts_norm_is_zero_width(tts_norm_op op) {
|
||||
return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
|
||||
|| op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
|
||||
}
|
||||
|
||||
//
|
||||
// parser
|
||||
//
|
||||
|
||||
struct tts_norm_parser {
|
||||
std::vector<uint32_t> s;
|
||||
size_t p = 0;
|
||||
size_t & n_caps;
|
||||
|
||||
tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
|
||||
|
||||
[[noreturn]] void fail(const std::string & msg) const {
|
||||
throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
|
||||
}
|
||||
|
||||
bool eof() const { return p >= s.size(); }
|
||||
uint32_t peek() const { return s[p]; }
|
||||
bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
|
||||
|
||||
void expect(uint32_t c, const char * what) {
|
||||
if (!peek_is(c)) {
|
||||
fail(std::string("expected ") + what);
|
||||
}
|
||||
p++;
|
||||
}
|
||||
|
||||
std::vector<tts_norm_seq> parse_alts() {
|
||||
std::vector<tts_norm_seq> alts;
|
||||
alts.push_back(parse_seq());
|
||||
while (peek_is('|')) {
|
||||
p++;
|
||||
alts.push_back(parse_seq());
|
||||
}
|
||||
return alts;
|
||||
}
|
||||
|
||||
tts_norm_seq parse_seq() {
|
||||
tts_norm_seq seq;
|
||||
while (!eof() && peek() != '|' && peek() != ')') {
|
||||
tts_norm_item item;
|
||||
item.atom = parse_atom();
|
||||
parse_quantifier(item);
|
||||
seq.push_back(std::move(item));
|
||||
}
|
||||
return seq;
|
||||
}
|
||||
|
||||
int parse_int() {
|
||||
if (eof() || peek() < '0' || peek() > '9') {
|
||||
fail("expected number in quantifier");
|
||||
}
|
||||
int v = 0;
|
||||
while (!eof() && peek() >= '0' && peek() <= '9') {
|
||||
v = v * 10 + static_cast<int>(peek() - '0');
|
||||
if (v > TTS_NORM_MAX_REPEAT) {
|
||||
fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
|
||||
}
|
||||
p++;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
void parse_quantifier(tts_norm_item & item) {
|
||||
if (eof()) {
|
||||
return;
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c != '?' && c != '*' && c != '+' && c != '{') {
|
||||
return;
|
||||
}
|
||||
if (tts_norm_is_zero_width(item.atom.op)) {
|
||||
fail("zero-width assertion cannot be repeated");
|
||||
}
|
||||
p++;
|
||||
if (c == '?') {
|
||||
item.min = 0; item.max = 1;
|
||||
} else if (c == '*') {
|
||||
item.min = 0; item.max = TTS_NORM_INF;
|
||||
} else if (c == '+') {
|
||||
item.min = 1; item.max = TTS_NORM_INF;
|
||||
} else {
|
||||
item.min = parse_int();
|
||||
item.max = item.min;
|
||||
if (peek_is(',')) {
|
||||
p++;
|
||||
item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
|
||||
}
|
||||
expect('}', "'}' to close quantifier");
|
||||
if (item.max < item.min) {
|
||||
fail("quantifier max is smaller than min");
|
||||
}
|
||||
}
|
||||
if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
|
||||
fail("multiple quantifiers");
|
||||
}
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_group() {
|
||||
// '(' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::GROUP;
|
||||
if (peek_is('?')) {
|
||||
p++;
|
||||
if (eof()) {
|
||||
fail("unterminated group");
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c == ':') {
|
||||
p++;
|
||||
} else if (c == '=' || c == '!') {
|
||||
p++;
|
||||
node.op = tts_norm_op::LOOKAHEAD;
|
||||
node.neg = c == '!';
|
||||
} else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
|
||||
node.op = tts_norm_op::LOOKBEHIND;
|
||||
node.neg = s[p + 1] == '!';
|
||||
p += 2;
|
||||
tts_norm_item item;
|
||||
if (eof() || peek() == ')') {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
item.atom = parse_atom();
|
||||
if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
if (!peek_is(')')) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
p++;
|
||||
node.alts.push_back(tts_norm_seq{});
|
||||
node.alts[0].push_back(std::move(item));
|
||||
return node;
|
||||
} else if (c == 'i') {
|
||||
fail("(?i) is only allowed at the start of the pattern");
|
||||
} else {
|
||||
fail("unknown group type");
|
||||
}
|
||||
} else {
|
||||
node.capture = static_cast<int>(n_caps++);
|
||||
}
|
||||
node.alts = parse_alts();
|
||||
expect(')', "')' to close group");
|
||||
return node;
|
||||
}
|
||||
|
||||
// after '%' inside or outside of a set
|
||||
// returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
|
||||
bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
|
||||
boundary = false;
|
||||
if (eof()) {
|
||||
fail("dangling '%'");
|
||||
}
|
||||
const uint32_t e = s[p++];
|
||||
if (tts_norm_is_class_letter(e)) {
|
||||
cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
|
||||
cls.neg = e >= 'A' && e <= 'Z';
|
||||
return true;
|
||||
}
|
||||
if (e == 'b' && !in_set) {
|
||||
boundary = true;
|
||||
return false;
|
||||
}
|
||||
if (tts_norm_is_ascii_alnum(e)) {
|
||||
p--;
|
||||
fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
|
||||
}
|
||||
lit = e;
|
||||
return false;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_set() {
|
||||
// '[' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::SET;
|
||||
if (peek_is('^')) {
|
||||
node.set.neg = true;
|
||||
p++;
|
||||
}
|
||||
bool empty = true;
|
||||
while (true) {
|
||||
if (eof()) {
|
||||
fail("unterminated set");
|
||||
}
|
||||
if (peek() == ']') {
|
||||
p++;
|
||||
break;
|
||||
}
|
||||
empty = false;
|
||||
uint32_t lo = 0;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, lo, boundary)) {
|
||||
node.set.classes.push_back(cls);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
lo = s[p++];
|
||||
}
|
||||
uint32_t hi = lo;
|
||||
if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
|
||||
p++;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, hi, boundary)) {
|
||||
fail("character class cannot be used in a range");
|
||||
}
|
||||
} else {
|
||||
hi = s[p++];
|
||||
}
|
||||
if (hi < lo) {
|
||||
fail("invalid range in set");
|
||||
}
|
||||
}
|
||||
node.set.ranges.emplace_back(lo, hi);
|
||||
}
|
||||
if (empty) {
|
||||
fail("empty set");
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_atom() {
|
||||
const uint32_t c = s[p++];
|
||||
tts_norm_pattern_node node;
|
||||
switch (c) {
|
||||
case '(':
|
||||
return parse_group();
|
||||
case '[':
|
||||
return parse_set();
|
||||
case '.':
|
||||
node.op = tts_norm_op::ANY;
|
||||
return node;
|
||||
case '^':
|
||||
node.op = tts_norm_op::START;
|
||||
return node;
|
||||
case '$':
|
||||
node.op = tts_norm_op::END;
|
||||
return node;
|
||||
case '%': {
|
||||
tts_norm_class cls{};
|
||||
uint32_t lit = 0;
|
||||
bool boundary = false;
|
||||
if (parse_escape(false, cls, lit, boundary)) {
|
||||
node.op = tts_norm_op::SET;
|
||||
node.set.classes.push_back(cls);
|
||||
} else if (boundary) {
|
||||
node.op = tts_norm_op::BOUNDARY;
|
||||
} else {
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = lit;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
case '?':
|
||||
case '*':
|
||||
case '+':
|
||||
case '{':
|
||||
p--;
|
||||
fail("nothing to repeat");
|
||||
case ']':
|
||||
case '}':
|
||||
p--;
|
||||
fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
|
||||
default:
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = c;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// matcher
|
||||
// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
|
||||
// (they never give back what they consumed), so there is no backtracking
|
||||
//
|
||||
|
||||
struct tts_norm_matcher {
|
||||
const std::vector<uint32_t> & t;
|
||||
bool icase;
|
||||
std::vector<std::pair<int64_t, int64_t>> caps;
|
||||
|
||||
bool is_w(size_t i) const {
|
||||
return i < t.size() && tts_norm_class_match('w', t[i]);
|
||||
}
|
||||
|
||||
bool set_match(const tts_norm_set & st, uint32_t c) const {
|
||||
auto test = [&](uint32_t x) {
|
||||
for (const auto & r : st.ranges) {
|
||||
if (x >= r.first && x <= r.second) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (const auto & k : st.classes) {
|
||||
if (tts_norm_class_match(k.kind, x) != k.neg) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
bool m = test(c);
|
||||
if (!m && icase) {
|
||||
const uint32_t o = tts_norm_ascii_swapcase(c);
|
||||
if (o != c) {
|
||||
m = test(o);
|
||||
}
|
||||
}
|
||||
return m != st.neg;
|
||||
}
|
||||
|
||||
bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
|
||||
for (const auto & seq : alts) {
|
||||
auto saved = caps;
|
||||
if (match_seq(seq, i, end)) {
|
||||
return true;
|
||||
}
|
||||
caps = std::move(saved);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
|
||||
size_t pos = i;
|
||||
for (const auto & it : seq) {
|
||||
int count = 0;
|
||||
while (count < it.max) {
|
||||
const bool is_group = it.atom.op == tts_norm_op::GROUP;
|
||||
std::vector<std::pair<int64_t, int64_t>> saved;
|
||||
if (is_group) {
|
||||
saved = caps;
|
||||
}
|
||||
size_t e = pos;
|
||||
if (!match_once(it.atom, pos, e)) {
|
||||
if (is_group) {
|
||||
caps = std::move(saved);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (e == pos) {
|
||||
// an empty iteration would repeat forever, it satisfies any remaining minimum
|
||||
count = std::max(count + 1, it.min);
|
||||
break;
|
||||
}
|
||||
pos = e;
|
||||
count++;
|
||||
}
|
||||
if (count < it.min) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
end = pos;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
|
||||
const size_t n = t.size();
|
||||
switch (node.op) {
|
||||
case tts_norm_op::LITERAL:
|
||||
if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::SET:
|
||||
if (i < n && set_match(node.set, t[i])) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::ANY:
|
||||
if (i < n) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::START:
|
||||
end = i;
|
||||
return i == 0;
|
||||
case tts_norm_op::END:
|
||||
end = i;
|
||||
return i == n;
|
||||
case tts_norm_op::BOUNDARY:
|
||||
end = i;
|
||||
return (i > 0 && is_w(i - 1)) != is_w(i);
|
||||
case tts_norm_op::GROUP: {
|
||||
size_t e = i;
|
||||
if (!match_alts(node.alts, i, e)) {
|
||||
return false;
|
||||
}
|
||||
if (node.capture >= 0) {
|
||||
caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
|
||||
}
|
||||
end = e;
|
||||
return true;
|
||||
}
|
||||
case tts_norm_op::LOOKAHEAD: {
|
||||
// captures made inside a lookahead are discarded
|
||||
auto saved = caps;
|
||||
size_t e = i;
|
||||
const bool ok = match_alts(node.alts, i, e);
|
||||
caps = std::move(saved);
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
case tts_norm_op::LOOKBEHIND: {
|
||||
size_t e = 0;
|
||||
const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// common_tts_norm_pattern
|
||||
//
|
||||
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
|
||||
if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
|
||||
throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
|
||||
}
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(src, cpts, offs);
|
||||
|
||||
tts_norm_parser parser(std::move(cpts), n_caps);
|
||||
static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
|
||||
if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
|
||||
icase = true;
|
||||
parser.p = 4;
|
||||
}
|
||||
|
||||
root = std::make_unique<tts_norm_pattern_node>();
|
||||
root->op = tts_norm_op::GROUP;
|
||||
root->alts = parser.parse_alts();
|
||||
if (!parser.eof()) {
|
||||
parser.fail("unbalanced ')'");
|
||||
}
|
||||
}
|
||||
|
||||
common_tts_norm_pattern::~common_tts_norm_pattern() = default;
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
|
||||
common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
|
||||
|
||||
bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
|
||||
tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
|
||||
size_t e = pos;
|
||||
if (!m.match_alts(root->alts, pos, e)) {
|
||||
return false;
|
||||
}
|
||||
end = e;
|
||||
groups = std::move(m.caps);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t common_tts_norm_pattern::n_groups() const {
|
||||
return n_caps;
|
||||
}
|
||||
|
||||
//
|
||||
// scan / sub
|
||||
//
|
||||
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(text, cpts, offs);
|
||||
|
||||
std::vector<common_tts_norm_segment> out;
|
||||
const size_t n = cpts.size();
|
||||
size_t i = 0;
|
||||
size_t pending = 0; // start of the current unmatched run
|
||||
std::vector<std::pair<int64_t, int64_t>> groups;
|
||||
|
||||
while (i < n) {
|
||||
bool matched = false;
|
||||
for (size_t r = 0; r < rules.size(); r++) {
|
||||
size_t end = i;
|
||||
if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
|
||||
if (pending < i) {
|
||||
out.push_back({-1, offs[pending], offs[i], {}});
|
||||
}
|
||||
common_tts_norm_segment seg;
|
||||
seg.rule = static_cast<int>(r);
|
||||
seg.start = offs[i];
|
||||
seg.end = offs[end];
|
||||
for (const auto & g : groups) {
|
||||
if (g.first < 0) {
|
||||
seg.groups.emplace_back(std::string::npos, std::string::npos);
|
||||
} else {
|
||||
seg.groups.emplace_back(offs[g.first], offs[g.second]);
|
||||
}
|
||||
}
|
||||
out.push_back(std::move(seg));
|
||||
i = end;
|
||||
pending = i;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (pending < n) {
|
||||
out.push_back({-1, offs[pending], offs[n], {}});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
struct tts_norm_repl_piece {
|
||||
std::string literal;
|
||||
int group = -1; // 0 = whole match, 1.. = groups
|
||||
};
|
||||
|
||||
static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
|
||||
std::vector<tts_norm_repl_piece> pieces;
|
||||
std::string lit;
|
||||
auto flush = [&]() {
|
||||
if (!lit.empty()) {
|
||||
pieces.push_back({lit, -1});
|
||||
lit.clear();
|
||||
}
|
||||
};
|
||||
for (size_t i = 0; i < repl.size(); i++) {
|
||||
if (repl[i] != '%') {
|
||||
lit += repl[i];
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= repl.size()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
|
||||
}
|
||||
const char c = repl[++i];
|
||||
if (c == '%') {
|
||||
lit += '%';
|
||||
} else if (c >= '0' && c <= '9') {
|
||||
const int g = c - '0';
|
||||
if (static_cast<size_t>(g) > pattern.n_groups()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
|
||||
}
|
||||
flush();
|
||||
pieces.push_back({"", g});
|
||||
} else {
|
||||
throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return pieces;
|
||||
}
|
||||
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
|
||||
const auto pieces = tts_norm_parse_repl(repl, pattern);
|
||||
std::string out;
|
||||
for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
|
||||
if (seg.rule < 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
continue;
|
||||
}
|
||||
for (const auto & piece : pieces) {
|
||||
if (piece.group < 0) {
|
||||
out += piece.literal;
|
||||
} else if (piece.group == 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
} else {
|
||||
const auto & g = seg.groups[piece.group - 1];
|
||||
if (g.first != std::string::npos) {
|
||||
out.append(text, g.first, g.second - g.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
//
|
||||
// template builtins
|
||||
//
|
||||
|
||||
// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
|
||||
static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
|
||||
std::vector<std::pair<std::string, std::string>> rules;
|
||||
if (is_val<value_string>(v)) {
|
||||
rules.emplace_back("match", v->as_string().str());
|
||||
} else if (is_val<value_object>(v)) {
|
||||
for (const auto & kv : v->as_ordered_object()) {
|
||||
if (!is_val<value_string>(kv.second)) {
|
||||
throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
|
||||
}
|
||||
rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
|
||||
}
|
||||
} else if (is_val<value_array>(v)) {
|
||||
for (const auto & item : v->as_array()) {
|
||||
if (!is_val<value_array>(item) || item->as_array().size() != 2
|
||||
|| !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
|
||||
throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
|
||||
}
|
||||
rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
|
||||
}
|
||||
} else {
|
||||
throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_scan(const func_args & args) {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_val<value_string>(args.get_pos(0));
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const std::string text = text_val.str();
|
||||
const bool is_input = text_val.all_parts_are_input();
|
||||
|
||||
const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
|
||||
std::vector<common_tts_norm_pattern> patterns;
|
||||
patterns.reserve(rule_srcs.size());
|
||||
std::vector<const common_tts_norm_pattern *> ptrs;
|
||||
for (const auto & r : rule_srcs) {
|
||||
patterns.emplace_back(r.second);
|
||||
}
|
||||
for (const auto & p : patterns) {
|
||||
ptrs.push_back(&p);
|
||||
}
|
||||
|
||||
auto mk_str = [&](size_t start, size_t end) -> value {
|
||||
return mk_val<value_string>(string(text.substr(start, end - start), is_input));
|
||||
};
|
||||
|
||||
auto out = mk_val<value_array>();
|
||||
for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
|
||||
auto groups = mk_val<value_array>();
|
||||
for (const auto & g : seg.groups) {
|
||||
// groups that did not participate are empty strings, so they render as nothing and are falsy
|
||||
groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
|
||||
}
|
||||
auto obj = mk_val<value_object>();
|
||||
obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
|
||||
obj->insert("text", mk_str(seg.start, seg.end));
|
||||
obj->insert("groups", groups);
|
||||
out->push_back(obj);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_sub(const func_args & args) {
|
||||
args.ensure_count(3, 3);
|
||||
args.ensure_vals<value_string, value_string, value_string>();
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
|
||||
const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value) {
|
||||
const auto it = std::lower_bound(dict.keys.begin(), dict.keys.end(), key);
|
||||
if (it == dict.keys.end() || *it != key) {
|
||||
return false;
|
||||
}
|
||||
const size_t idx = it - dict.keys.begin();
|
||||
if (idx >= dict.values.size()) {
|
||||
throw std::runtime_error("tts_dict: malformed dictionary, a key has no value");
|
||||
}
|
||||
value = dict.values[idx];
|
||||
return true;
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts, common_tts_g2p g2p) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_vals<value_string, value_string>();
|
||||
const std::string name = args.get_pos(0)->as_string().str();
|
||||
const auto it = dicts ? dicts->find(name) : common_tts_dicts::const_iterator();
|
||||
if (!dicts || it == dicts->end()) {
|
||||
throw std::runtime_error("tts_dict: unknown dictionary '" + name + "'");
|
||||
}
|
||||
std::string doc;
|
||||
if (!common_tts_dict_find(it->second, args.get_pos(1)->as_string().str(), doc)) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return value_from_json(common_json::parse(doc), false);
|
||||
}));
|
||||
// none when the caller gives no letter-to-sound model, or the model has no pronunciation
|
||||
ctx.set_val("tts_g2p", mk_val<value_func>("tts_g2p", [g2p](const func_args & args) -> value {
|
||||
args.ensure_count(1, 1);
|
||||
args.ensure_vals<value_string>();
|
||||
const std::string out = g2p ? g2p(args.get_pos(0)->as_string().str()) : std::string();
|
||||
if (out.empty()) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return mk_val<value_string>(out);
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
|
||||
//
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// functions exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
// tts_dict(name, key) -> value of key in a dictionary of the model, or none
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
}
|
||||
|
||||
struct tts_norm_pattern_node;
|
||||
|
||||
class common_tts_norm_pattern {
|
||||
public:
|
||||
// throws std::runtime_error on invalid pattern
|
||||
explicit common_tts_norm_pattern(const std::string & src);
|
||||
~common_tts_norm_pattern();
|
||||
common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
|
||||
common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
|
||||
|
||||
// match anchored at codepoint index pos; on success, end is the codepoint index after the match
|
||||
// groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
|
||||
bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
|
||||
|
||||
size_t n_groups() const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<tts_norm_pattern_node> root;
|
||||
size_t n_caps = 0;
|
||||
bool icase = false;
|
||||
};
|
||||
|
||||
// segment produced by common_tts_norm_scan(); byte offsets into the input text
|
||||
struct common_tts_norm_segment {
|
||||
int rule = -1; // index of the matched rule, -1 for unmatched text
|
||||
size_t start = 0;
|
||||
size_t end = 0;
|
||||
std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
|
||||
};
|
||||
|
||||
// split text into matched and unmatched segments
|
||||
// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
|
||||
|
||||
// replace every non-overlapping match of pattern with repl
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// a dictionary of the model, e.g. a pronunciation lexicon
|
||||
struct common_tts_dict {
|
||||
std::vector<std::string> keys; // sorted by their bytes
|
||||
std::vector<std::string> values; // the JSON document of each key
|
||||
};
|
||||
|
||||
using common_tts_dicts = std::map<std::string, common_tts_dict>;
|
||||
|
||||
// binary search of key in a dictionary; on success, value is set to the JSON document
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
|
||||
|
||||
// pronunciation of a word by a letter-to-sound model, as phonemes separated by spaces (e.g. CMUdict notation)
|
||||
// empty when there is none; the notation is the one of the model, its "tts" template knows it
|
||||
using common_tts_g2p = std::function<std::string(const std::string & word)>;
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr,
|
||||
common_tts_g2p g2p = nullptr);
|
||||
+53
-279
@@ -3,7 +3,6 @@
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "chat-tts-norm.h"
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
@@ -191,17 +190,10 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
|
||||
} else {
|
||||
auto & parts = jmsg["content"] = json::array();
|
||||
for (const auto & part : content_parts) {
|
||||
if (part.type == "tool_reference") {
|
||||
parts.push_back({
|
||||
{"type", part.type},
|
||||
{"name", part.name},
|
||||
});
|
||||
} else {
|
||||
parts.push_back({
|
||||
{"type", part.type},
|
||||
{"text", part.text},
|
||||
});
|
||||
}
|
||||
parts.push_back({
|
||||
{"type", part.type},
|
||||
{"text", part.text},
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -317,8 +309,6 @@ struct common_chat_templates {
|
||||
bool has_explicit_template; // Model had builtin template or template overridden was specified.
|
||||
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
|
||||
std::unique_ptr<common_chat_template> template_tool_use;
|
||||
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
|
||||
std::shared_ptr<common_tts_dicts> tts_dicts; // dictionaries of the model for tts_dict(), by name
|
||||
};
|
||||
|
||||
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
|
||||
@@ -380,16 +370,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
|
||||
throw std::invalid_argument("Missing content part type: " + part.dump());
|
||||
}
|
||||
const auto & type = part.at("type");
|
||||
common_chat_msg_content_part msg_part;
|
||||
msg_part.type = type;
|
||||
if (type == "text" || type == "media_marker") {
|
||||
msg_part.text = part.at("text");
|
||||
} else if (type == "tool_reference") {
|
||||
// names a request tool, the template renders its definition
|
||||
msg_part.name = part.at("name");
|
||||
} else {
|
||||
if (type != "text" && type != "media_marker") {
|
||||
throw std::invalid_argument("Unsupported content part type: " + type.dump());
|
||||
}
|
||||
common_chat_msg_content_part msg_part;
|
||||
msg_part.type = type;
|
||||
msg_part.text = part.at("text");
|
||||
msg.content_parts.push_back(msg_part);
|
||||
}
|
||||
} else if (!content.is_null()) {
|
||||
@@ -442,41 +428,6 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
|
||||
if (message.contains("tool_call_id")) {
|
||||
msg.tool_call_id = message.at("tool_call_id");
|
||||
}
|
||||
{
|
||||
json extra = json::object();
|
||||
for (const auto & [k, v] : message.items()) {
|
||||
if (k != "role" && k != "content" && k != "tool_calls" && k != "reasoning_content" &&
|
||||
k != "name" && k != "tool_call_id") {
|
||||
extra[k] = v;
|
||||
}
|
||||
}
|
||||
if (has_tool_calls) {
|
||||
// per tool call: keys besides type/function/id, and function keys besides name/arguments
|
||||
json tc_extra = json::array();
|
||||
bool any = false;
|
||||
for (const auto & tool_call : message.at("tool_calls")) {
|
||||
json e = json::object();
|
||||
for (const auto & [k, v] : tool_call.items()) {
|
||||
if (k != "type" && k != "function" && k != "id") {
|
||||
e[k] = v;
|
||||
}
|
||||
}
|
||||
for (const auto & [k, v] : tool_call.at("function").items()) {
|
||||
if (k != "name" && k != "arguments") {
|
||||
e["function"][k] = v;
|
||||
}
|
||||
}
|
||||
any = any || !e.empty();
|
||||
tc_extra.push_back(e);
|
||||
}
|
||||
if (any) {
|
||||
extra["__tool_call_extras"] = tc_extra;
|
||||
}
|
||||
}
|
||||
if (!extra.empty()) {
|
||||
msg.extra_json = extra.dump();
|
||||
}
|
||||
}
|
||||
|
||||
msgs.push_back(msg);
|
||||
}
|
||||
@@ -565,50 +516,6 @@ static json render_message_to_json(const std::vector<common_chat_msg> & msgs, co
|
||||
return messages_inp_normalizer(c).normalize(messages);
|
||||
}
|
||||
|
||||
// DeepSeek V4.1 reads non-OpenAI message keys and joins content blocks with "\n\n" around images,
|
||||
// so keep media markers as separate text parts and restore the extra keys
|
||||
static json render_message_to_json_deepseek_v4_1(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {
|
||||
json messages = json::array();
|
||||
for (const auto & msg : msgs) {
|
||||
json jmsg;
|
||||
if (msg.contains_media() && msg.content.empty()) {
|
||||
common_chat_msg typed = msg;
|
||||
for (auto & part : typed.content_parts) {
|
||||
part.type = "text";
|
||||
}
|
||||
jmsg = typed.to_json_oaicompat(/* concat_typed_text= */ false);
|
||||
} else {
|
||||
jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
|
||||
}
|
||||
if (!msg.extra_json.empty()) {
|
||||
const json extra = json::parse(msg.extra_json);
|
||||
for (const auto & [k, v] : extra.items()) {
|
||||
if (k == "__tool_call_extras") {
|
||||
if (!jmsg.contains("tool_calls")) {
|
||||
continue;
|
||||
}
|
||||
auto & tcs = jmsg.at("tool_calls");
|
||||
for (size_t i = 0; i < tcs.size() && i < v.size(); i++) {
|
||||
for (const auto & [tk, tv] : v[i].items()) {
|
||||
if (tk == "function") {
|
||||
for (const auto & [fk, fv] : tv.items()) {
|
||||
tcs[i]["function"][fk] = fv;
|
||||
}
|
||||
} else {
|
||||
tcs[i][tk] = tv;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (!jmsg.contains(k)) {
|
||||
jmsg[k] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
messages.push_back(std::move(jmsg));
|
||||
}
|
||||
return messages_inp_normalizer(c).normalize(messages);
|
||||
}
|
||||
|
||||
// DEPRECATED: only used in tests
|
||||
json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {
|
||||
jinja::caps c;
|
||||
@@ -624,17 +531,13 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
|
||||
|
||||
auto result = json::array();
|
||||
for (const auto & tool : tools) {
|
||||
json function {
|
||||
{ "name", tool.name },
|
||||
{ "description", tool.description },
|
||||
{ "parameters", json::parse(tool.parameters) },
|
||||
};
|
||||
if (tool.defer_loading) {
|
||||
function["defer_loading"] = true;
|
||||
}
|
||||
result.push_back({
|
||||
{ "type", "function" },
|
||||
{ "function", function },
|
||||
{ "function", {
|
||||
{ "name", tool.name },
|
||||
{ "description", tool.description },
|
||||
{ "parameters", json::parse(tool.parameters) },
|
||||
}},
|
||||
});
|
||||
}
|
||||
return result;
|
||||
@@ -671,19 +574,10 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
|
||||
}
|
||||
|
||||
const auto & function = tool.at("function");
|
||||
|
||||
// "defer_loading" sits on the function, or on the tool next to "type"; the function wins
|
||||
const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
|
||||
: tool.value("defer_loading", json(false));
|
||||
if (!defer.is_boolean()) {
|
||||
throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
|
||||
}
|
||||
|
||||
result.push_back({
|
||||
/* .name = */ function.at("name"),
|
||||
/* .description = */ function.value("description", ""),
|
||||
/* .parameters = */ function.value("parameters", json::object()).dump(),
|
||||
/* .defer_loading = */ defer.get<bool>(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -806,119 +700,19 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
return tmpls->has_explicit_template;
|
||||
}
|
||||
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts,
|
||||
const common_tts_g2p & g2p = nullptr) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts, g2p);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
jinja::runtime runtime(ctx);
|
||||
return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
|
||||
}
|
||||
|
||||
// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
|
||||
static bool is_lfm3_template(const std::string & src) {
|
||||
return src.find("<additional_tools>") != std::string::npos;
|
||||
}
|
||||
|
||||
static std::string string_lower(std::string s) {
|
||||
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); });
|
||||
return s;
|
||||
}
|
||||
|
||||
// VibeVoice-ASR: default system prompt of the template written by the converter
|
||||
static bool is_vibevoice_asr_template(const std::string & src) {
|
||||
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
{ "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
|
||||
{ "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
|
||||
{ "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
|
||||
{ "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
|
||||
};
|
||||
const auto it = names.find(string_lower(language));
|
||||
return it != names.end() ? it->second : language;
|
||||
}
|
||||
|
||||
// LFM3-Audio: no language = auto detect
|
||||
static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
if (language.empty()) {
|
||||
return "Perform ASR.";
|
||||
}
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
|
||||
double audio_length, const std::string & context) {
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
|
||||
common_chat_prompt_preset asr_preset;
|
||||
asr_preset.system = "";
|
||||
asr_preset.user = "Transcribe audio to text";
|
||||
|
||||
if (chat_templates && chat_templates->template_default) {
|
||||
const std::string src = chat_templates->template_default->source();
|
||||
if (is_lfm2_template(src)) {
|
||||
asr_preset.system = "Perform ASR.";
|
||||
asr_preset.user = "";
|
||||
} else if (is_lfm3_template(src)) {
|
||||
asr_preset.system = lfm3_asr_system_prompt(language);
|
||||
asr_preset.user = "";
|
||||
asr_preset.has_language = true;
|
||||
} else if (is_vibevoice_asr_template(src)) {
|
||||
// without the audio length the model does not know where the audio ends and loops
|
||||
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
|
||||
if (context.empty()) {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
|
||||
} else {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
|
||||
asr_preset.has_context = true;
|
||||
}
|
||||
asr_preset.media_first = true;
|
||||
asr_preset.needs_audio_length = true;
|
||||
}
|
||||
if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
|
||||
asr_preset.system = "Perform ASR.";
|
||||
asr_preset.user = "";
|
||||
}
|
||||
|
||||
return asr_preset;
|
||||
}
|
||||
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language, const std::string & voice, const common_tts_g2p & g2p) {
|
||||
if (!chat_templates || !chat_templates->template_tts) {
|
||||
return "";
|
||||
}
|
||||
return common_chat_template_render(*chat_templates->template_tts, {
|
||||
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
|
||||
{"language", language},
|
||||
{"voice", voice},
|
||||
}, chat_templates->tts_dicts, g2p);
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
const std::string & language, const std::string & voice) {
|
||||
if (!chat_templates || !chat_templates->template_default || !is_lfm3_template(chat_templates->template_default->source())) {
|
||||
return "";
|
||||
}
|
||||
// same voices as the "tts" template
|
||||
const std::string v = string_lower(voice);
|
||||
const bool has_voice = v == "female" || v == "male";
|
||||
if (language.empty() && !has_voice) {
|
||||
return "";
|
||||
}
|
||||
std::string out = "Speak";
|
||||
if (!language.empty()) {
|
||||
out += " in " + lfm3_language_name(language);
|
||||
}
|
||||
if (has_voice) {
|
||||
out += " using a " + v + " voice";
|
||||
}
|
||||
return out + ".";
|
||||
}
|
||||
|
||||
std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
|
||||
if (!variant.empty()) {
|
||||
if (variant == "tool_use") {
|
||||
@@ -974,6 +768,18 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
"{%- if false %}");
|
||||
}
|
||||
|
||||
// TODO @aldehir : this is a temporary fix, pending Minja changes
|
||||
// Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664
|
||||
if (default_template_src.find("[TOOL_CALLS]") != std::string::npos
|
||||
// search for the error message and patch it
|
||||
&& default_template_src.find("if (message['content'] is none or") != std::string::npos) {
|
||||
string_replace_all(default_template_src,
|
||||
"{%- if (message['content'] is none or message['content'] == '' or "
|
||||
"message['content']|length == 0) and (message['tool_calls'] is not defined or "
|
||||
"message['tool_calls'] is none or message['tool_calls']|length == 0) %}",
|
||||
"{%- if false %}");
|
||||
}
|
||||
|
||||
std::string token_bos = bos_token_override;
|
||||
std::string token_eos = eos_token_override;
|
||||
bool add_bos = false;
|
||||
@@ -1017,35 +823,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// task templates of the model, kept with a chat template override
|
||||
if (model) {
|
||||
if (const char * src = llama_model_chat_template(model, "tts")) {
|
||||
try {
|
||||
tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// dictionaries for tts_dict(), listed by name in tokenizer.tts_dicts
|
||||
tmpls->tts_dicts = std::make_shared<common_tts_dicts>();
|
||||
for (int32_t i = 0; i < llama_model_meta_arr_n(model, "tokenizer.tts_dicts"); i++) {
|
||||
const std::string name = llama_model_meta_arr_str(model, "tokenizer.tts_dicts", i);
|
||||
const std::string key = "tokenizer.tts_dict." + name + ".keys";
|
||||
const std::string val_key = "tokenizer.tts_dict." + name + ".values";
|
||||
const int32_t n = llama_model_meta_arr_n(model, key.c_str());
|
||||
if (n < 0 || llama_model_meta_arr_n(model, val_key.c_str()) != n) {
|
||||
LOG_ERR("%s: the tts dictionary '%s' has no value for each of its keys (ignoring it)\n", __func__, name.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & dict = (*tmpls->tts_dicts)[name];
|
||||
dict.keys.reserve(n);
|
||||
dict.values.reserve(n);
|
||||
for (int32_t j = 0; j < n; j++) {
|
||||
dict.keys.emplace_back(llama_model_meta_arr_str(model, key.c_str(), j));
|
||||
dict.values.emplace_back(llama_model_meta_arr_str(model, val_key.c_str(), j));
|
||||
}
|
||||
}
|
||||
}
|
||||
return tmpls;
|
||||
}
|
||||
|
||||
@@ -1303,6 +1080,13 @@ static common_chat_params common_chat_params_init_lfm2_5(const common_chat_templ
|
||||
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
|
||||
}
|
||||
|
||||
// Older gemma4 templates need their tool responses rewritten before rendering
|
||||
static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
|
||||
auto adjusted = inputs;
|
||||
workaround::convert_tool_responses_gemma4(adjusted.messages);
|
||||
return common_chat_params_init_gemma4(tmpl, adjusted);
|
||||
}
|
||||
|
||||
// Pick the dedicated handler for a template from its source, or null for the autoparser.
|
||||
// Order matters: the first match wins, and later checks assume the earlier ones did not match.
|
||||
static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
|
||||
@@ -1424,19 +1208,22 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
|
||||
// The template source contains the token as a variable assignment, not as a literal in markup.
|
||||
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
|
||||
// V4.1 tag names have a leading space and its template does not name the dsml_token variable.
|
||||
if ((src.find("dsml_token") != std::string::npos &&
|
||||
src.find("DSML") != std::string::npos &&
|
||||
(src.find("function_calls") != std::string::npos ||
|
||||
src.find("tool_calls") != std::string::npos)) ||
|
||||
src.find("|DSML| calls>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: DeepSeek V3.2/V4/V4.1\n");
|
||||
if (src.find("dsml_token") != std::string::npos &&
|
||||
src.find("DSML") != std::string::npos &&
|
||||
(src.find("function_calls") != std::string::npos ||
|
||||
src.find("tool_calls") != std::string::npos)) {
|
||||
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
|
||||
return common_chat_params_init_deepseek_v3_2;
|
||||
}
|
||||
|
||||
// Gemma4 format detection
|
||||
if (src.find("'<|tool_call>call:'") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Gemma4\n");
|
||||
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
|
||||
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
|
||||
"Consider updating to the official template.\n", __func__);
|
||||
return common_chat_params_init_gemma4_legacy;
|
||||
}
|
||||
return common_chat_params_init_gemma4;
|
||||
}
|
||||
|
||||
@@ -1465,12 +1252,6 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
return common_chat_params_init_qwen3_coder;
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
if (is_inkling_template(src)) {
|
||||
LOG_DBG("Using specialized template: Inkling\n");
|
||||
return common_chat_params_init_inkling;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -1532,9 +1313,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
workaround::trim_all_content(trimmed_messages);
|
||||
messages_to_render = &trimmed_messages;
|
||||
}
|
||||
params.messages = src.find("|DSML| calls>") != std::string::npos ?
|
||||
render_message_to_json_deepseek_v4_1(*messages_to_render, tmpl.original_caps()) :
|
||||
render_message_to_json(*messages_to_render, tmpl.original_caps());
|
||||
params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps());
|
||||
params.tool_choice = inputs.tool_choice;
|
||||
params.reasoning_format = inputs.reasoning_format;
|
||||
params.enable_thinking = inputs.enable_thinking;
|
||||
@@ -1855,9 +1634,6 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
|
||||
|
||||
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
|
||||
if (!is_partial) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_COMPLETE;
|
||||
}
|
||||
if (params.debug) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
|
||||
}
|
||||
@@ -1995,14 +1771,12 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch
|
||||
return result;
|
||||
}
|
||||
|
||||
common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
|
||||
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
|
||||
GGML_ASSERT(chat_templates != nullptr);
|
||||
GGML_ASSERT(chat_templates->template_default != nullptr);
|
||||
// take the more expressive template when available
|
||||
const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
|
||||
: chat_templates->template_default->caps;
|
||||
common_json result = caps.to_map();
|
||||
result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
|
||||
result["reasoning_effort_rejected"] = caps.reasoning_effort_rejected;
|
||||
return result;
|
||||
if (chat_templates->template_tool_use != nullptr) {
|
||||
// take the more expressive template when available
|
||||
return chat_templates->template_tool_use->caps.to_map();
|
||||
}
|
||||
return chat_templates->template_default->caps.to_map();
|
||||
}
|
||||
+3
-27
@@ -45,7 +45,6 @@ struct common_chat_tool_call {
|
||||
struct common_chat_msg_content_part {
|
||||
std::string type;
|
||||
std::string text;
|
||||
std::string name; // tool_reference: name of a tool from the request tools
|
||||
|
||||
// TODO @ngxson : no known chat templates support reasoning_content in content parts yet
|
||||
// this can be useful for models with interleaved thinking (like Kimi-K2)
|
||||
@@ -53,7 +52,7 @@ struct common_chat_msg_content_part {
|
||||
// std::string reasoning_content;
|
||||
|
||||
bool operator==(const common_chat_msg_content_part & other) const {
|
||||
return type == other.type && text == other.text && name == other.name;
|
||||
return type == other.type && text == other.text;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -95,8 +94,6 @@ struct common_chat_msg {
|
||||
std::string reasoning_content;
|
||||
std::string tool_name;
|
||||
std::string tool_call_id;
|
||||
// non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
|
||||
std::string extra_json;
|
||||
|
||||
common_json to_json_oaicompat(bool concat_typed_text = false) const;
|
||||
|
||||
@@ -227,9 +224,6 @@ struct common_chat_tool {
|
||||
std::string name;
|
||||
std::string description;
|
||||
std::string parameters;
|
||||
|
||||
// withheld from the system prompt, a tool_reference reveals it later in the conversation
|
||||
bool defer_loading = false;
|
||||
};
|
||||
|
||||
enum common_chat_tool_choice {
|
||||
@@ -472,7 +466,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
|
||||
common_json common_chat_tool_parameters(const common_json & function);
|
||||
|
||||
// get template caps, useful for reporting to server /props endpoint
|
||||
common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
|
||||
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
|
||||
|
||||
common_chat_prompt common_chat_template_direct_apply(
|
||||
const common_chat_template & tmpl,
|
||||
@@ -491,24 +485,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
struct common_chat_prompt_preset {
|
||||
std::string system;
|
||||
std::string user;
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_context = false; // the user text already holds the context
|
||||
bool media_first = false; // the media goes before the user text
|
||||
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
|
||||
};
|
||||
|
||||
// audio_length: in seconds, 0 if unknown
|
||||
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
|
||||
double audio_length = 0.0, const std::string & context = "");
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
// g2p (optional): the letter-to-sound model of the template's tts_g2p(), e.g. mtmd_gen_audio_g2p()
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language = "", const std::string & voice = "",
|
||||
const std::function<std::string(const std::string & word)> & g2p = nullptr);
|
||||
|
||||
// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
const std::string & language, const std::string & voice);
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
|
||||
+130
-188
@@ -219,23 +219,6 @@ int32_t common_cpu_get_num_math() {
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(__riscv) && defined(__linux__)
|
||||
std::ifstream cpuinfo("/proc/cpuinfo");
|
||||
std::string line;
|
||||
int32_t n_a100 = 0;
|
||||
while (std::getline(cpuinfo, line)) {
|
||||
std::istringstream fields(line);
|
||||
std::string key;
|
||||
std::string separator;
|
||||
uint64_t marchid;
|
||||
if (fields >> key >> separator >> std::hex >> marchid &&
|
||||
key == "marchid" && separator == ":" && marchid == UINT64_C(0x8000000041000002)) {
|
||||
++n_a100;
|
||||
}
|
||||
}
|
||||
if (n_a100 > 0) {
|
||||
return n_a100;
|
||||
}
|
||||
#elif defined(__powerpc64__) || defined(__powerpc__)
|
||||
int32_t smt_factor = 1;
|
||||
int phy_cpus = common_cpu_get_num_physical_cores();
|
||||
@@ -759,56 +742,50 @@ static inline bool glob_class_match(const char c, const char * pattern, const ch
|
||||
}
|
||||
|
||||
// simple glob: * matches non-/ chars, ** matches anything including /, [] matches character class
|
||||
// only wildcards recurse, too many wildcards: no match
|
||||
static bool glob_match(const char * pattern, const char * str, int depth = 0) {
|
||||
if (depth >= COMMON_RECURSIVE_LIMIT) {
|
||||
static inline bool glob_match(const char * pattern, const char * str) {
|
||||
if (*pattern == '\0') {
|
||||
return *str == '\0';
|
||||
}
|
||||
if (pattern[0] == '*' && pattern[1] == '*') {
|
||||
const char * p = pattern + 2;
|
||||
if (glob_match(p, str)) return true;
|
||||
if (*str != '\0') return glob_match(pattern, str + 1);
|
||||
return false;
|
||||
}
|
||||
while (*pattern != '\0') {
|
||||
if (pattern[0] == '*' && pattern[1] == '*') {
|
||||
const char * p = pattern + 2;
|
||||
for (;; str++) {
|
||||
if (glob_match(p, str, depth + 1)) return true;
|
||||
if (*str == '\0') return false;
|
||||
}
|
||||
if (*pattern == '*') {
|
||||
const char * p = pattern + 1;
|
||||
for (; *str != '\0' && *str != '/'; str++) {
|
||||
if (glob_match(p, str)) return true;
|
||||
}
|
||||
if (*pattern == '*') {
|
||||
const char * p = pattern + 1;
|
||||
for (; *str != '\0' && *str != '/'; str++) {
|
||||
if (glob_match(p, str, depth + 1)) return true;
|
||||
}
|
||||
return glob_match(p, str, depth + 1);
|
||||
}
|
||||
if (*pattern == '?' && *str != '\0' && *str != '/') {
|
||||
pattern++;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
if (*pattern == '[') {
|
||||
const char * class_end = pattern + 1;
|
||||
// If first character after '[' is ']' or '-', treat it as literal
|
||||
if (*class_end == ']' || *class_end == '-') {
|
||||
class_end++;
|
||||
}
|
||||
while (*class_end != '\0' && *class_end != ']') {
|
||||
class_end++;
|
||||
}
|
||||
if (*class_end == ']') {
|
||||
if (*str == '\0' || !glob_class_match(*str, pattern + 1, class_end)) return false;
|
||||
pattern = class_end + 1;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
if (*str != '[') return false;
|
||||
pattern++;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
if (*pattern != *str) return false;
|
||||
pattern++;
|
||||
str++;
|
||||
return glob_match(p, str);
|
||||
}
|
||||
return *str == '\0';
|
||||
if (*pattern == '?' && *str != '\0' && *str != '/') {
|
||||
return glob_match(pattern + 1, str + 1);
|
||||
}
|
||||
if (*pattern == '[') {
|
||||
const char * class_end = pattern + 1;
|
||||
// If first character after '[' is ']' or '-', treat it as literal
|
||||
if (*class_end == ']' || *class_end == '-') {
|
||||
class_end++;
|
||||
}
|
||||
while (*class_end != '\0' && *class_end != ']') {
|
||||
class_end++;
|
||||
}
|
||||
if (*class_end == ']') {
|
||||
if (*str == '\0') return false;
|
||||
bool matched = glob_class_match(*str, pattern + 1, class_end);
|
||||
return matched && glob_match(class_end + 1, str + 1);
|
||||
} else {
|
||||
if (*str == '[') {
|
||||
return glob_match(pattern + 1, str + 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (*pattern == *str) {
|
||||
return glob_match(pattern + 1, str + 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool glob_match(const std::string & pattern, const std::string & str) {
|
||||
@@ -895,7 +872,27 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
|
||||
return true;
|
||||
}
|
||||
|
||||
#include <iostream>
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
std::wstring utf8_to_wstring(const std::string & str) {
|
||||
if (str.empty()) {
|
||||
return std::wstring();
|
||||
}
|
||||
|
||||
int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);
|
||||
|
||||
if (size <= 0) {
|
||||
return std::wstring();
|
||||
}
|
||||
|
||||
std::wstring wstr(size, 0);
|
||||
MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);
|
||||
|
||||
return wstr;
|
||||
}
|
||||
|
||||
std::string wstring_to_utf8(const std::wstring & str) {
|
||||
if (str.empty()) {
|
||||
return std::string();
|
||||
@@ -914,6 +911,12 @@ std::string wstring_to_utf8(const std::wstring & str) {
|
||||
}
|
||||
#endif
|
||||
|
||||
// returns the path as a UTF-8 string, preserving its separators
|
||||
std::string fs_path_to_utf8(const std::filesystem::path & path) {
|
||||
const auto value = path.u8string();
|
||||
return std::string(value.begin(), value.end());
|
||||
}
|
||||
|
||||
void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
|
||||
// a random suffix gives every writer its own temporary file, across processes and containers
|
||||
std::random_device rd;
|
||||
@@ -937,7 +940,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
|
||||
|
||||
if (file.fail() || ec) {
|
||||
std::filesystem::remove(path_tmp, ec);
|
||||
throw std::runtime_error("failed to write file: " + path.string());
|
||||
throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -963,9 +966,20 @@ void common_set_env(const std::string & name, const std::string & value) {
|
||||
#endif
|
||||
}
|
||||
|
||||
std::filesystem::path common_get_path_from_env(const std::string & name) {
|
||||
#if defined(_WIN32)
|
||||
const std::wstring wname = utf8_to_wstring(name);
|
||||
const wchar_t * wvalue = _wgetenv(wname.c_str());
|
||||
return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
|
||||
#else
|
||||
const char * value = std::getenv(name.c_str());
|
||||
return value ? std::filesystem::path(value) : std::filesystem::path();
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
static std::filesystem::path get_home_directory() {
|
||||
std::filesystem::path home = common_get_env("HOME");
|
||||
std::filesystem::path home = common_get_path_from_env("HOME");
|
||||
if (!home.empty()) {
|
||||
return home;
|
||||
}
|
||||
@@ -978,19 +992,19 @@ static std::filesystem::path get_home_directory() {
|
||||
#endif
|
||||
|
||||
std::filesystem::path fs_get_cache_directory() {
|
||||
std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE");
|
||||
std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
|
||||
if (!cache_directory.empty()) {
|
||||
return cache_directory;
|
||||
}
|
||||
#if defined(_WIN32)
|
||||
cache_directory = common_get_env("LOCALAPPDATA");
|
||||
cache_directory = common_get_path_from_env("LOCALAPPDATA");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
cache_directory = get_home_directory() / "Library/Caches";
|
||||
#else
|
||||
cache_directory = common_get_env("XDG_CACHE_HOME");
|
||||
cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
|
||||
if (cache_directory.empty()) {
|
||||
cache_directory = get_home_directory() / ".cache";
|
||||
}
|
||||
@@ -1001,12 +1015,12 @@ std::filesystem::path fs_get_cache_directory() {
|
||||
std::filesystem::path fs_get_config_directory() {
|
||||
std::filesystem::path config_directory;
|
||||
#if defined(_WIN32)
|
||||
config_directory = common_get_env("APPDATA");
|
||||
config_directory = common_get_path_from_env("APPDATA");
|
||||
if (config_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %APPDATA% directory");
|
||||
}
|
||||
#else
|
||||
config_directory = common_get_env("XDG_CONFIG_HOME");
|
||||
config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
|
||||
if (config_directory.empty()) {
|
||||
config_directory = get_home_directory() / ".config";
|
||||
}
|
||||
@@ -1020,9 +1034,9 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) {
|
||||
std::error_code ec;
|
||||
common_create_directories(cache_directory, ec);
|
||||
if (ec) {
|
||||
throw std::runtime_error("failed to create cache directory: " + cache_directory.string());
|
||||
throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
|
||||
}
|
||||
return cache_directory / filename;
|
||||
return cache_directory / std::filesystem::u8path(filename);
|
||||
}
|
||||
|
||||
//
|
||||
@@ -1167,9 +1181,6 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
|
||||
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
|
||||
{ COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
|
||||
{ COMMON_DECISION_TYPE_H2O_LIGHTNING, "h2o-lightning" },
|
||||
{ COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
|
||||
{ COMMON_DECISION_TYPE_LFM3_D1, "lfm3-d1" },
|
||||
{ COMMON_DECISION_TYPE_LFM3_D1_OMNI, "lfm3-d1-omni" },
|
||||
// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
|
||||
};
|
||||
|
||||
@@ -1235,52 +1246,46 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
|
||||
return info;
|
||||
}
|
||||
|
||||
void common_fit_apply(common_params & params, llama_model_params & mparams, llama_context_params & cparams) {
|
||||
if (!params.fit_params) {
|
||||
return;
|
||||
}
|
||||
|
||||
COM_TRC("%s", "fitting params to device memory ...\n");
|
||||
COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
|
||||
|
||||
// the draft context is created from the same base params and follows the main context, fit both together
|
||||
const bool has_draft = params.speculative.has_dft();
|
||||
const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
|
||||
common_params params_dft = common_base_params_to_speculative(params);
|
||||
|
||||
auto mparams_dft = common_model_params_to_llama(params_dft);
|
||||
auto cparams_dft = common_context_params_to_llama(params_dft);
|
||||
if (spec_mtp) {
|
||||
cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
|
||||
}
|
||||
cparams_dft.n_rs_seq = 0;
|
||||
|
||||
const common_fit_extra_model extra = {
|
||||
/*.paths_model =*/ params_dft.model.get_paths(),
|
||||
/*.mparams =*/ &mparams_dft,
|
||||
/*.cparams =*/ &cparams_dft,
|
||||
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
|
||||
};
|
||||
|
||||
common_fit_params(params.model.get_paths(), &mparams, &cparams,
|
||||
params.tensor_split,
|
||||
params.tensor_buft_overrides.data(),
|
||||
params.fit_params_target.data(),
|
||||
params.fit_params_min_ctx,
|
||||
has_draft || spec_mtp ? &extra : nullptr,
|
||||
params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
|
||||
}
|
||||
|
||||
common_init_result::common_init_result(common_params & params, bool model_only) :
|
||||
pimpl(new impl{}) {
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
||||
common_fit_apply(params, mparams, cparams);
|
||||
if (params.fit_params) {
|
||||
COM_TRC("%s", "fitting params to device memory ...\n");
|
||||
COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
|
||||
|
||||
llama_model * model = common_model_load(params.model.get_paths(), mparams);
|
||||
// the draft context is created from the same base params and follows the main context, fit both together
|
||||
const bool has_draft = params.speculative.has_dft();
|
||||
const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
|
||||
common_params params_dft = common_base_params_to_speculative(params);
|
||||
|
||||
auto mparams_dft = common_model_params_to_llama(params_dft);
|
||||
auto cparams_dft = common_context_params_to_llama(params_dft);
|
||||
if (spec_mtp) {
|
||||
cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
|
||||
}
|
||||
cparams_dft.n_rs_seq = 0;
|
||||
|
||||
const common_fit_extra_model extra = {
|
||||
/*.path_model =*/ params_dft.model.path.c_str(),
|
||||
/*.mparams =*/ &mparams_dft,
|
||||
/*.cparams =*/ &cparams_dft,
|
||||
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
|
||||
};
|
||||
|
||||
common_fit_params(params.model.path.c_str(), &mparams, &cparams,
|
||||
params.tensor_split,
|
||||
params.tensor_buft_overrides.data(),
|
||||
params.fit_params_target.data(),
|
||||
params.fit_params_min_ctx,
|
||||
has_draft || spec_mtp ? &extra : nullptr,
|
||||
params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
|
||||
}
|
||||
|
||||
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
|
||||
if (model == NULL) {
|
||||
return;
|
||||
}
|
||||
@@ -1296,34 +1301,19 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
// these decision models return a score for each token via the embeddings output
|
||||
// TODO: maybe improve this in the future
|
||||
const auto decision_type = common_get_decision_type(model);
|
||||
// minicpm-ocr-jev reads its class scores from the pooled output, the pooling type is the one of the GGUF
|
||||
if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
|
||||
decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2 ||
|
||||
decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
|
||||
const auto pooling_type = decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV ? LLAMA_POOLING_TYPE_UNSPECIFIED : LLAMA_POOLING_TYPE_NONE;
|
||||
|
||||
decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
|
||||
params.embedding = true;
|
||||
params.pooling_type = pooling_type;
|
||||
params.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
|
||||
cparams.embeddings = true;
|
||||
cparams.pooling_type = pooling_type;
|
||||
cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
cparams.n_outputs_max = cparams.n_batch;
|
||||
cparams.n_outputs_max_per_seq = 1;
|
||||
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// 0 is n_batch
|
||||
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
|
||||
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
|
||||
}
|
||||
|
||||
if (llama_model_classifier_free_guidance(model)) {
|
||||
params.kv_unified = true;
|
||||
cparams.kv_unified = true;
|
||||
cparams.n_seq_max *= 2;
|
||||
}
|
||||
|
||||
// embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch
|
||||
// (server.cpp does this check for --embedding, but before the model is loaded)
|
||||
if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) {
|
||||
@@ -1464,7 +1454,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
|
||||
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);
|
||||
|
||||
const auto cvec = common_control_vector_load(params.control_vectors, llama_model_n_layer(model));
|
||||
const auto cvec = common_control_vector_load(params.control_vectors);
|
||||
if (cvec.n_embd == -1) {
|
||||
return res;
|
||||
}
|
||||
@@ -1727,19 +1717,6 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
|
||||
return mparams;
|
||||
}
|
||||
|
||||
struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams) {
|
||||
if (paths.size() == 1) {
|
||||
return llama_model_load_from_file(paths[0].c_str(), mparams);
|
||||
}
|
||||
|
||||
std::vector<const char *> c_paths;
|
||||
c_paths.reserve(paths.size());
|
||||
for (const auto & path : paths) {
|
||||
c_paths.push_back(path.c_str());
|
||||
}
|
||||
return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams);
|
||||
}
|
||||
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params) {
|
||||
auto cparams = llama_context_default_params();
|
||||
|
||||
@@ -1776,8 +1753,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
|
||||
cparams.type_k = params.cache_type_k;
|
||||
cparams.type_v = params.cache_type_v;
|
||||
|
||||
cparams.moe_cache_size = params.moe_cache_size;
|
||||
cparams.moe_cache_cpu_share = params.moe_cache_cpu_share;
|
||||
cparams.moe_cache_size = params.moe_cache_size;
|
||||
|
||||
return cparams;
|
||||
}
|
||||
@@ -2000,7 +1976,7 @@ float common_embd_similarity_cos(const float * embd1, const float * embd2, int n
|
||||
// Control vector utils
|
||||
//
|
||||
|
||||
static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info, int32_t n_layer) {
|
||||
static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info) {
|
||||
common_control_vector_data result = { -1, {} };
|
||||
|
||||
ggml_context * ctx = nullptr;
|
||||
@@ -2041,10 +2017,6 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str());
|
||||
result.n_embd = -1;
|
||||
break;
|
||||
} else if (layer_idx > n_layer) {
|
||||
COM_ERR("direction tensor layer index %d exceeds the model layer count %d in %s\n", layer_idx, n_layer, load_info.fname.c_str());
|
||||
result.n_embd = -1;
|
||||
break;
|
||||
}
|
||||
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
|
||||
@@ -2068,10 +2040,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
}
|
||||
|
||||
// extend if necessary - do not store data for layer 0 (it's not used)
|
||||
result.data.resize(std::max(result.data.size(), (size_t) result.n_embd * layer_idx), 0.0f);
|
||||
result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
|
||||
|
||||
const float * src = (const float *) tensor->data;
|
||||
float * dst = result.data.data() + (size_t) result.n_embd * (layer_idx - 1); // layer 1 at [0]
|
||||
float * dst = result.data.data() + result.n_embd * (layer_idx - 1); // layer 1 at [0]
|
||||
for (int j = 0; j < result.n_embd; j++) {
|
||||
dst[j] += src[j] * load_info.strength; // allows multiple directions for same layer in same file
|
||||
}
|
||||
@@ -2089,11 +2061,11 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
return result;
|
||||
}
|
||||
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer) {
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos) {
|
||||
common_control_vector_data result = { -1, {} };
|
||||
|
||||
for (const auto & info : load_infos) {
|
||||
auto cur = common_control_vector_load_one(info, n_layer);
|
||||
auto cur = common_control_vector_load_one(info);
|
||||
|
||||
if (cur.n_embd == -1) {
|
||||
result.n_embd = -1;
|
||||
@@ -2260,14 +2232,6 @@ bool common_batch::set_embd_state(int32_t idx, llama_embd state) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_batch::set_embd_state_row(int32_t idx, int32_t row) {
|
||||
if (idx < 0 || idx >= size() || row < 0 || tokens[idx].state.data != nullptr || tokens[idx].state_row >= 0) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].state_row = row;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, { nullptr, 0, 0 }, {} };
|
||||
for (int32_t j = 0; j < n_pos; ++j) {
|
||||
@@ -2277,16 +2241,6 @@ int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq
|
||||
return size() - 1;
|
||||
}
|
||||
|
||||
int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) {
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} };
|
||||
t.embd_row = row;
|
||||
for (int32_t j = 0; j < n_pos; ++j) {
|
||||
t.pos[j] = pos[j];
|
||||
}
|
||||
tokens.push_back(t);
|
||||
return size() - 1;
|
||||
}
|
||||
|
||||
llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
GGML_ASSERT(batch && "common_batch was not initialized with a context");
|
||||
GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
|
||||
@@ -2307,12 +2261,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
|
||||
GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
|
||||
}
|
||||
} else if (t.embd_row >= 0) {
|
||||
idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row);
|
||||
if (idx < 0) {
|
||||
GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n);
|
||||
}
|
||||
llama_batch_ext_set_pos(res, idx, t.pos.data());
|
||||
} else {
|
||||
idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
|
||||
if (idx < 0) {
|
||||
@@ -2333,15 +2281,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.state.data) {
|
||||
llama_batch_ext_set_embd_state(res, idx, t.state); // contexts without a state input ignore it
|
||||
}
|
||||
if (t.state_row >= 0 && !llama_batch_ext_set_embd_state_row(res, idx, t.state_row)) {
|
||||
GGML_ABORT("%s: failed to set the state row of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
|
||||
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
+24
-81
@@ -31,8 +31,6 @@
|
||||
#define DIRECTORY_SEPARATOR '/'
|
||||
#endif // _WIN32
|
||||
|
||||
#define COMMON_RECURSIVE_LIMIT 128
|
||||
|
||||
#define COM_DBG(fmt, ...) LOG_DBG("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)
|
||||
#define COM_TRC(fmt, ...) LOG_TRC("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)
|
||||
#define COM_INF(fmt, ...) LOG_INF("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)
|
||||
@@ -174,20 +172,18 @@ enum common_params_sampling_config : uint64_t {
|
||||
};
|
||||
|
||||
enum common_speculative_type {
|
||||
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2, // DFlash2 speculative decoding (DFlash + selector)
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR, // LiliCorr speculative decoding (DFlash + scorer)
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
|
||||
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MOD,
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
|
||||
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
|
||||
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
|
||||
};
|
||||
|
||||
// Grammar type enumeration
|
||||
@@ -313,18 +309,6 @@ struct common_params_model {
|
||||
std::string hf_file = ""; // HF file
|
||||
std::string docker_repo = ""; // Docker repo
|
||||
|
||||
std::vector<std::string> shards;
|
||||
|
||||
std::vector<std::string> get_paths() const {
|
||||
std::vector<std::string> paths = { path };
|
||||
for (const auto & shard : shards) {
|
||||
if (std::find(paths.begin(), paths.end(), shard) == paths.end()) {
|
||||
paths.push_back(shard);
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
std::string get_name() const {
|
||||
if (!hf_repo.empty()) {
|
||||
return hf_repo;
|
||||
@@ -416,12 +400,7 @@ struct common_params_speculative {
|
||||
|
||||
uint32_t need_n_rs_seq() const {
|
||||
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2 ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR;
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
|
||||
});
|
||||
|
||||
return needs_rs_seq ? draft.n_max : 0u;
|
||||
@@ -616,8 +595,7 @@ struct common_params {
|
||||
ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K
|
||||
ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V
|
||||
|
||||
size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers
|
||||
float moe_cache_cpu_share = 0.0f; // share of the MoE cache misses computed on the CPU instead of uploaded
|
||||
size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers
|
||||
|
||||
common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO;
|
||||
|
||||
@@ -704,7 +682,6 @@ struct common_params {
|
||||
// UI configs
|
||||
bool ui = true;
|
||||
bool ui_mcp_proxy = false;
|
||||
bool ui_mcp_proxy_allow_private_urls = false; // allow the MCP CORS proxy to reach local or private addresses
|
||||
std::string ui_config_json;
|
||||
|
||||
// "advanced" endpoints are disabled by default for better security
|
||||
@@ -715,7 +692,6 @@ struct common_params {
|
||||
// enable built-in tools
|
||||
std::vector<std::string> server_tools;
|
||||
std::string server_tools_runtime;
|
||||
bool server_tools_allow_unauth = false; // allow tools/MCP on a non-loopback address without API key
|
||||
|
||||
// MCP server configs (Cursor-compatible JSON)
|
||||
std::string mcp_servers_config; // path to JSON file with MCP server definitions
|
||||
@@ -725,7 +701,6 @@ struct common_params {
|
||||
std::string models_dir = ""; // directory containing models for the router server
|
||||
std::string models_preset = ""; // directory containing model presets for the router server
|
||||
int models_max = 4; // maximum number of models to load simultaneously
|
||||
int models_memory_margin = 0; // MiB of device memory to keep free when loading models (0 = no memory accounting)
|
||||
bool models_autoload = true; // automatically load models when requested via the router server
|
||||
std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty)
|
||||
|
||||
@@ -733,7 +708,6 @@ struct common_params {
|
||||
|
||||
std::string slot_save_path;
|
||||
std::string media_path; // path to directory for loading media files
|
||||
bool media_allow_private_urls = false; // allow http(s) media URLs on local or private addresses
|
||||
|
||||
float slot_prompt_similarity = 0.1f;
|
||||
|
||||
@@ -797,9 +771,7 @@ struct common_params {
|
||||
|
||||
// TTS params
|
||||
std::string tts_lang = "";
|
||||
std::string tts_voice = "";
|
||||
std::string tts_speaker_file = "";
|
||||
std::string tts_speaker_text = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
@@ -932,9 +904,13 @@ bool glob_match(const std::string & pattern, const std::string & str);
|
||||
//
|
||||
|
||||
#ifdef _WIN32
|
||||
std::string wstring_to_utf8(const std::wstring & str);
|
||||
std::wstring utf8_to_wstring(const std::string & str);
|
||||
std::string wstring_to_utf8(const std::wstring & str);
|
||||
#endif
|
||||
|
||||
// returns the path as a UTF-8 string, preserving its separators
|
||||
std::string fs_path_to_utf8(const std::filesystem::path & path);
|
||||
|
||||
//
|
||||
// Environment utils
|
||||
//
|
||||
@@ -944,6 +920,9 @@ std::string wstring_to_utf8(const std::wstring & str);
|
||||
std::string common_get_env(const std::string & name);
|
||||
void common_set_env(const std::string & name, const std::string & value);
|
||||
|
||||
// reads a path from the environment, an unset variable gives an empty path
|
||||
std::filesystem::path common_get_path_from_env(const std::string & name);
|
||||
|
||||
//
|
||||
// Filesystem utils
|
||||
//
|
||||
@@ -999,15 +978,9 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
|
||||
COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
|
||||
COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
|
||||
COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
|
||||
COMMON_DECISION_TYPE_LFM3_D1, // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
|
||||
COMMON_DECISION_TYPE_LFM3_D1_OMNI, // same as lfm3-d1
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
// lfm3-d1: max questions packed in one prompt
|
||||
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// metadata of a GGUF file, read without loading the model
|
||||
@@ -1044,11 +1017,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
struct llama_model_params common_model_params_to_llama ( common_params & params);
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params);
|
||||
|
||||
// apply --fit to mparams and cparams, no-op if params.fit_params is false
|
||||
void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
|
||||
|
||||
struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams);
|
||||
|
||||
// clear LoRA adapters from context, then apply new list of adapters
|
||||
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
|
||||
|
||||
@@ -1123,9 +1091,6 @@ struct common_batch {
|
||||
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
|
||||
@@ -1164,17 +1129,10 @@ struct common_batch {
|
||||
// attach a state embedding (e.g. the target hidden state for MTP) to the entry at idx, can only be set once per entry
|
||||
bool set_embd_state(int32_t idx, llama_embd state);
|
||||
|
||||
// the state of entry idx is row `row` of the h_nextn device buffer of the state source context, see llama_set_ctx_state_src()
|
||||
bool set_embd_state_row(int32_t idx, int32_t row);
|
||||
|
||||
// add an embedding-only entry (no token id)
|
||||
// pos points to n_pos positions
|
||||
int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
// add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other
|
||||
// pos points to n_pos positions
|
||||
int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
int32_t size() const { return (int32_t) tokens.size(); }
|
||||
};
|
||||
|
||||
@@ -1271,9 +1229,8 @@ struct common_control_vector_load_info {
|
||||
};
|
||||
|
||||
// Load control vectors, scale each by strength, and add them together.
|
||||
// Direction layer indices must be in [1, n_layer].
|
||||
// On error, returns {-1, empty}
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer);
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);
|
||||
|
||||
//
|
||||
// Split utils
|
||||
@@ -1325,20 +1282,6 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
|
||||
// prompt utils
|
||||
//
|
||||
|
||||
// leave default-inserted elements uninitialized; callers must fill them before use
|
||||
template <typename T>
|
||||
struct common_no_init_allocator : std::allocator<T> {
|
||||
template <typename U> struct rebind { using other = common_no_init_allocator<U>; };
|
||||
|
||||
common_no_init_allocator() = default;
|
||||
template <typename U> common_no_init_allocator(const common_no_init_allocator<U> &) {}
|
||||
|
||||
template <typename U> void construct(U * p) { ::new ((void *) p) U; }
|
||||
};
|
||||
|
||||
// sequence state data, filled by llama_state_seq_get_data_ext()
|
||||
using common_state_data = std::vector<uint8_t, common_no_init_allocator<uint8_t>>;
|
||||
|
||||
struct common_prompt_checkpoint {
|
||||
int64_t n_tokens;
|
||||
|
||||
@@ -1348,8 +1291,8 @@ struct common_prompt_checkpoint {
|
||||
llama_pos pos_min;
|
||||
llama_pos pos_max;
|
||||
|
||||
common_state_data data_tgt;
|
||||
common_state_data data_dft;
|
||||
std::vector<uint8_t> data_tgt;
|
||||
std::vector<uint8_t> data_dft;
|
||||
|
||||
// (optional) speculative-decoding implementation state stashed with the checkpoint
|
||||
// (e.g. eagle3's deferred-boundary g_embd row)
|
||||
|
||||
+36
-53
@@ -45,7 +45,7 @@ static const std::string PARTIAL_SUFFIX = ".downloadInProgress";
|
||||
|
||||
static void write_etag(const std::string & path, const std::string & etag) {
|
||||
const std::string etag_path = path + ".etag";
|
||||
fs_write_atomic(etag_path, etag);
|
||||
fs_write_atomic(std::filesystem::u8path(etag_path), etag);
|
||||
LOG_DBG("%s: file etag saved: %s\n", __func__, etag_path.c_str());
|
||||
}
|
||||
|
||||
@@ -293,7 +293,8 @@ static int common_download_file_single_online(const std::string & url,
|
||||
|
||||
// alt_path is the HF snapshot: the blob may have been moved there when symlinks are unavailable
|
||||
auto cache_hit = [&] {
|
||||
return std::filesystem::exists(path) || (!alt_path.empty() && std::filesystem::exists(alt_path));
|
||||
return std::filesystem::exists(std::filesystem::u8path(path)) ||
|
||||
(!alt_path.empty() && std::filesystem::exists(std::filesystem::u8path(alt_path)));
|
||||
};
|
||||
|
||||
if (skip_etag && cache_hit()) {
|
||||
@@ -461,55 +462,38 @@ static int common_download_file_single_online(const std::string & url,
|
||||
return finish(success ? head->status : -1);
|
||||
}
|
||||
|
||||
std::pair<long, std::vector<char>> common_remote_get_content(const std::string & url_in,
|
||||
std::pair<long, std::vector<char>> common_remote_get_content(const std::string & url,
|
||||
const common_remote_params & params) {
|
||||
static constexpr int max_redirects = 5;
|
||||
std::string url = url_in;
|
||||
for (int n_redirects = 0; ; n_redirects++) {
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
|
||||
httplib::Headers headers;
|
||||
for (const auto & h : params.headers) {
|
||||
headers.emplace(h.first, h.second);
|
||||
}
|
||||
if (headers.find("User-Agent") == headers.end()) {
|
||||
headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info()));
|
||||
}
|
||||
|
||||
if (params.timeout > 0) {
|
||||
cli.set_read_timeout(params.timeout, 0);
|
||||
cli.set_write_timeout(params.timeout, 0);
|
||||
}
|
||||
if (params.resolve_host) {
|
||||
// pin checked address, follow redirects manually to check each hop
|
||||
cli.set_direct_hostname_addr_map({{parts.host, params.resolve_host(parts.host)}});
|
||||
cli.set_follow_location(false);
|
||||
}
|
||||
|
||||
std::vector<char> buf;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char *data, size_t len) {
|
||||
buf.insert(buf.end(), data, data + len);
|
||||
return params.max_size == 0 ||
|
||||
buf.size() <= static_cast<size_t>(params.max_size);
|
||||
},
|
||||
nullptr
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw std::runtime_error("error: cannot make GET request");
|
||||
}
|
||||
|
||||
if (params.resolve_host && res->status >= 300 && res->status < 400 && res->has_header("Location")) {
|
||||
if (n_redirects >= max_redirects) {
|
||||
throw std::runtime_error("error: too many redirects");
|
||||
}
|
||||
url = common_http_resolve_location(url, res->get_header_value("Location"));
|
||||
continue;
|
||||
}
|
||||
|
||||
return { res->status, std::move(buf) };
|
||||
httplib::Headers headers;
|
||||
for (const auto & h : params.headers) {
|
||||
headers.emplace(h.first, h.second);
|
||||
}
|
||||
if (headers.find("User-Agent") == headers.end()) {
|
||||
headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info()));
|
||||
}
|
||||
|
||||
if (params.timeout > 0) {
|
||||
cli.set_read_timeout(params.timeout, 0);
|
||||
cli.set_write_timeout(params.timeout, 0);
|
||||
}
|
||||
|
||||
std::vector<char> buf;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char *data, size_t len) {
|
||||
buf.insert(buf.end(), data, data + len);
|
||||
return params.max_size == 0 ||
|
||||
buf.size() <= static_cast<size_t>(params.max_size);
|
||||
},
|
||||
nullptr
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw std::runtime_error("error: cannot make GET request");
|
||||
}
|
||||
|
||||
return { res->status, std::move(buf) };
|
||||
}
|
||||
|
||||
int common_download_file_single(const std::string & url,
|
||||
@@ -527,7 +511,7 @@ int common_download_file_single(const std::string & url,
|
||||
return common_download_file_single_online(url, path, online_opts, skip_etag, lock_path, alt_path);
|
||||
}
|
||||
|
||||
if (!std::filesystem::exists(path)) {
|
||||
if (!std::filesystem::exists(std::filesystem::u8path(path))) {
|
||||
LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
|
||||
return -1;
|
||||
}
|
||||
@@ -764,8 +748,7 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
|
||||
}
|
||||
|
||||
for (const auto & t : tags) {
|
||||
// match the tag as a full token, so ":F16" does not match a BF16 file
|
||||
std::regex pattern("(^|[-./])" + t + "[.-]", std::regex::icase);
|
||||
std::regex pattern(t + "[.-]", std::regex::icase);
|
||||
for (const auto & f : files) {
|
||||
if (gguf_filename_is_model(f.path) &&
|
||||
std::regex_search(f.path, pattern)) {
|
||||
@@ -1022,7 +1005,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
|
||||
std::string model_filename = repo;
|
||||
std::replace(model_filename.begin(), model_filename.end(), '/', '_');
|
||||
model_filename += "_" + tag + ".gguf";
|
||||
std::string local_path = fs_get_cache_file(model_filename).string();
|
||||
std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
|
||||
|
||||
const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
|
||||
common_download_opts opts;
|
||||
@@ -1228,7 +1211,7 @@ bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const
|
||||
for (const auto & s : sidecars) {
|
||||
const auto & f = plan.*s.file;
|
||||
if (!f.local_path.empty()) {
|
||||
to_remove.emplace_back(f.local_path);
|
||||
to_remove.emplace_back(std::filesystem::u8path(f.local_path));
|
||||
}
|
||||
}
|
||||
if (to_remove.empty()) {
|
||||
|
||||
@@ -31,8 +31,6 @@ struct common_remote_params {
|
||||
common_header_list headers;
|
||||
long timeout = 0; // in seconds, 0 means no timeout
|
||||
long max_size = 0; // unlimited if 0
|
||||
// if set: called per hop (redirects too), returns the address to connect to, throws to refuse
|
||||
std::function<std::string(const std::string & host)> resolve_host;
|
||||
};
|
||||
|
||||
// get remote file content, returns <http_code, raw_response_body>
|
||||
|
||||
+118
-233
@@ -1,8 +1,6 @@
|
||||
#include "fit.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "json.h"
|
||||
#include "llama-cpp.h"
|
||||
#include "log.h"
|
||||
|
||||
#include "../src/llama-ext.h"
|
||||
@@ -29,56 +27,46 @@ class common_params_fit_exception : public std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
// RAII: send llama logs below min_level to the debug log
|
||||
struct common_fit_log_scope {
|
||||
common_fit_log_scope(ggml_log_level min_level) : min_level(min_level) {
|
||||
llama_log_get(&original_logger.callback, &original_logger.user_data);
|
||||
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
|
||||
const common_fit_log_scope * self = (const common_fit_log_scope *) user_data;
|
||||
const ggml_log_level level_eff = level >= self->min_level ? level : GGML_LOG_LEVEL_DEBUG;
|
||||
self->original_logger.callback(level_eff, text, self->original_logger.user_data);
|
||||
}, this);
|
||||
}
|
||||
|
||||
~common_fit_log_scope() {
|
||||
llama_log_set(original_logger.callback, original_logger.user_data);
|
||||
}
|
||||
|
||||
common_fit_log_scope(const common_fit_log_scope &) = delete;
|
||||
common_fit_log_scope & operator=(const common_fit_log_scope &) = delete;
|
||||
|
||||
private:
|
||||
struct {
|
||||
ggml_log_callback callback;
|
||||
void * user_data;
|
||||
} original_logger;
|
||||
ggml_log_level min_level;
|
||||
};
|
||||
|
||||
static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
common_fit_log_scope log_scope(log_level);
|
||||
struct user_data_t {
|
||||
struct {
|
||||
ggml_log_callback callback;
|
||||
void * user_data;
|
||||
} original_logger;
|
||||
ggml_log_level min_level; // prints below this log level go to debug log
|
||||
};
|
||||
user_data_t ud;
|
||||
llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
|
||||
ud.min_level = log_level;
|
||||
|
||||
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
|
||||
const user_data_t * ud = (const user_data_t *) user_data;
|
||||
const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
|
||||
ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
|
||||
}, &ud);
|
||||
|
||||
llama_model_params mparams_copy = *mparams;
|
||||
mparams_copy.no_alloc = true;
|
||||
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model * model = common_model_load(paths_model, mparams_copy);
|
||||
llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
|
||||
if (model == nullptr) {
|
||||
llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
|
||||
throw std::runtime_error("failed to load model");
|
||||
}
|
||||
|
||||
llama_context * ctx = llama_init_from_model(model, *cparams);
|
||||
if (ctx == nullptr) {
|
||||
llama_model_free(model);
|
||||
llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
|
||||
throw std::runtime_error("failed to create llama_context from model");
|
||||
}
|
||||
|
||||
@@ -153,8 +141,6 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
if (mparams->load_mtp) {
|
||||
hp_ngl += llama_model_n_layer_nextn(model);
|
||||
}
|
||||
// without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers
|
||||
hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model);
|
||||
hp_n_ctx_train = llama_model_n_ctx_train(model);
|
||||
hp_n_expert = llama_model_n_expert(model);
|
||||
|
||||
@@ -162,93 +148,22 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
uint32_t * n_ctx_train,
|
||||
uint32_t * n_ctx,
|
||||
ggml_log_level log_level) {
|
||||
common_fit_log_scope log_scope(log_level);
|
||||
|
||||
llama_model_params mparams_copy = *mparams;
|
||||
mparams_copy.no_alloc = true;
|
||||
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model_ptr model(common_model_load(paths_model, mparams_copy));
|
||||
if (!model) {
|
||||
throw std::runtime_error("failed to load model");
|
||||
}
|
||||
|
||||
llama_context_ptr ctx(llama_init_from_model(model.get(), *cparams));
|
||||
if (!ctx) {
|
||||
throw std::runtime_error("failed to create llama_context from model");
|
||||
}
|
||||
|
||||
if (n_ctx_train) {
|
||||
*n_ctx_train = llama_model_n_ctx_train(model.get());
|
||||
}
|
||||
if (n_ctx) {
|
||||
*n_ctx = llama_n_ctx(ctx.get());
|
||||
}
|
||||
|
||||
return common_memory_breakdown_rows(ctx.get());
|
||||
}
|
||||
|
||||
common_memory_reports common_measure_memory(common_params & params) {
|
||||
const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
|
||||
const std::vector<std::string> paths_model = params.model.get_paths();
|
||||
|
||||
common_memory_reports res;
|
||||
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
||||
// max: what --fit picks now, no-op with -fit off
|
||||
{
|
||||
auto mparams_fit = mparams;
|
||||
auto cparams_fit = cparams;
|
||||
common_fit_apply(params, mparams_fit, cparams_fit);
|
||||
res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
|
||||
}
|
||||
|
||||
// min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
|
||||
uint32_t n_ctx_min = cparams.n_ctx;
|
||||
if (n_ctx_min == 0 && params.fit_params) {
|
||||
const uint32_t n_streams = cparams.kv_unified ? 1 : std::max<uint32_t>(1, cparams.n_seq_max);
|
||||
// -c 0 sets fit_params_min_ctx to UINT32_MAX, cap at n_ctx_train like --fit
|
||||
n_ctx_min = (uint32_t) std::min<uint64_t>(
|
||||
uint64_t(params.fit_params_min_ctx) * n_streams,
|
||||
uint64_t(res.n_ctx_train) * n_streams);
|
||||
}
|
||||
if (n_ctx_min == 0 || n_ctx_min >= res.n_ctx_max) {
|
||||
// nothing to reduce, or -fit off with the trained context
|
||||
res.n_ctx_min = res.n_ctx_max;
|
||||
res.info_min = res.info_max;
|
||||
} else {
|
||||
cparams.n_ctx = n_ctx_min;
|
||||
res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
common_device_memory_data_vec common_get_device_memory_data(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
|
||||
common_device_memory_data_vec ret(impl.size());
|
||||
for (size_t i = 0; i < impl.size(); i++) {
|
||||
@@ -262,7 +177,7 @@ common_device_memory_data_vec common_get_device_memory_data(
|
||||
}
|
||||
|
||||
static void common_params_fit_impl(
|
||||
const std::vector<std::string> & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
|
||||
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
|
||||
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
|
||||
size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {
|
||||
if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {
|
||||
@@ -274,7 +189,6 @@ static void common_params_fit_impl(
|
||||
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
@@ -296,7 +210,6 @@ static void common_params_fit_impl(
|
||||
if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {
|
||||
std::vector<ggml_backend_dev_t> devs_extra;
|
||||
uint32_t ngl_extra = 0;
|
||||
uint32_t nsk_extra = 0;
|
||||
uint32_t nct_extra = 0;
|
||||
uint32_t nex_extra = 0;
|
||||
|
||||
@@ -308,7 +221,7 @@ static void common_params_fit_impl(
|
||||
dmds_t measured;
|
||||
try {
|
||||
measured = common_get_device_memory_data_impl(
|
||||
extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
|
||||
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
|
||||
} catch (const std::runtime_error & e) {
|
||||
// the extra model is optional, fit the main model alone rather than giving up
|
||||
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
|
||||
@@ -348,7 +261,7 @@ static void common_params_fit_impl(
|
||||
// step 1: get data for default parameters and check whether any changes are necessary in the first place
|
||||
|
||||
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
|
||||
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
|
||||
@@ -360,7 +273,7 @@ static void common_params_fit_impl(
|
||||
if (n_streams > 1) {
|
||||
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
|
||||
__func__, n_ctx_max, n_streams);
|
||||
dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
}
|
||||
}
|
||||
add_extra_memory(dmds_full);
|
||||
@@ -499,7 +412,7 @@ static void common_params_fit_impl(
|
||||
|
||||
int64_t sum_projected_used_min_ctx = 0;
|
||||
cparams->n_ctx = n_ctx_min_total;
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_min_ctx);
|
||||
if (nd == 0) {
|
||||
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
|
||||
@@ -646,20 +559,6 @@ static void common_params_fit_impl(
|
||||
assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);
|
||||
uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides
|
||||
|
||||
// the skipped NextN layers come right before the output layer, put them on the same device as the output layer
|
||||
// so that the layers assigned by the model loader match the layers above
|
||||
if (hp_nsk > 0 && mparams.n_gpu_layers > 0) {
|
||||
mparams.n_gpu_layers += hp_nsk;
|
||||
if (nd > 1) {
|
||||
// the last devices have no layers if not even the output layer fits
|
||||
size_t id_last = nd - 1;
|
||||
while (ngl_per_device[id_last].n_layer == 0) {
|
||||
id_last--;
|
||||
}
|
||||
tensor_split[id_last] += hp_nsk;
|
||||
}
|
||||
}
|
||||
|
||||
mparams.tensor_split = tensor_split;
|
||||
|
||||
size_t itbo = 0;
|
||||
@@ -695,7 +594,7 @@ static void common_params_fit_impl(
|
||||
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
|
||||
|
||||
dmds_t dmd_nl = common_get_device_memory_data_impl(
|
||||
paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmd_nl);
|
||||
|
||||
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
|
||||
@@ -724,7 +623,7 @@ static void common_params_fit_impl(
|
||||
|
||||
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
|
||||
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_cpu_moe);
|
||||
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
@@ -978,7 +877,7 @@ static void common_params_fit_impl(
|
||||
}
|
||||
|
||||
enum common_params_fit_status common_fit_params(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams,
|
||||
float * tensor_split,
|
||||
@@ -990,7 +889,7 @@ enum common_params_fit_status common_fit_params(
|
||||
const int64_t t0_us = llama_time_us();
|
||||
common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
|
||||
try {
|
||||
common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
|
||||
common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
|
||||
LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
|
||||
} catch (const common_params_fit_exception & e) {
|
||||
LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
|
||||
@@ -1004,7 +903,8 @@ enum common_params_fit_status common_fit_params(
|
||||
return status;
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const struct llama_context * ctx) {
|
||||
void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
//const auto & devices = ctx->get_model().devices;
|
||||
const auto * model = llama_get_model(ctx);
|
||||
|
||||
std::vector<ggml_backend_dev_t> devices;
|
||||
@@ -1014,6 +914,18 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
|
||||
|
||||
llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);
|
||||
|
||||
std::vector<std::array<std::string, 9>> table_data;
|
||||
table_data.reserve(devices.size());
|
||||
|
||||
// same data as the table below, for --log-jsonl consumers
|
||||
common_json rows = common_json::array();
|
||||
const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
|
||||
const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
|
||||
const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
|
||||
|
||||
table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
|
||||
|
||||
constexpr size_t MiB = 1024 * 1024;
|
||||
const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};
|
||||
|
||||
// track seen buffer types to avoid double counting:
|
||||
@@ -1052,130 +964,104 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> rows;
|
||||
rows.reserve(devices.size() + 1 + memory_breakdown.size());
|
||||
|
||||
// print memory breakdown for each device:
|
||||
for (size_t i = 0; i < devices.size(); i++) {
|
||||
ggml_backend_dev_t dev = devices[i];
|
||||
llama_memory_breakdown_data mb = mb_dev[i];
|
||||
|
||||
common_memory_breakdown_row row;
|
||||
row.kind = common_memory_breakdown_row::DEVICE;
|
||||
row.name = ggml_backend_dev_name(dev);
|
||||
row.description = ggml_backend_dev_description(dev);
|
||||
const std::string name = ggml_backend_dev_name(dev);
|
||||
std::string desc = ggml_backend_dev_description(dev);
|
||||
for (const std::string & prefix : desc_prefixes_strip) {
|
||||
if (row.description.length() >= prefix.length() && row.description.substr(0, prefix.length()) == prefix) {
|
||||
row.description = row.description.substr(prefix.length());
|
||||
if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {
|
||||
desc = desc.substr(prefix.length());
|
||||
}
|
||||
}
|
||||
|
||||
size_t free, total;
|
||||
ggml_backend_dev_memory(dev, &free, &total);
|
||||
row.total = total;
|
||||
row.free = free;
|
||||
row.model = mb_dev[i].model;
|
||||
row.context = mb_dev[i].context;
|
||||
row.compute = mb_dev[i].compute;
|
||||
rows.push_back(std::move(row));
|
||||
|
||||
const size_t self = mb.model + mb.context + mb.compute;
|
||||
const int64_t unaccounted = static_cast<int64_t>(total) - static_cast<int64_t>(free) - static_cast<int64_t>(self);
|
||||
|
||||
table_data.push_back({
|
||||
template_gpu,
|
||||
" - " + name + " (" + desc + ")",
|
||||
std::to_string(total / MiB),
|
||||
std::to_string(free / MiB),
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(mb.model / MiB),
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
std::to_string(unaccounted / static_cast<int64_t>(MiB))});
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "device"},
|
||||
{"name", name},
|
||||
{"description", desc},
|
||||
{"total", total / MiB},
|
||||
{"free", free / MiB},
|
||||
{"self", self / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
{"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
|
||||
});
|
||||
}
|
||||
|
||||
// print memory breakdown for host:
|
||||
{
|
||||
common_memory_breakdown_row row;
|
||||
row.kind = common_memory_breakdown_row::HOST;
|
||||
row.name = "Host";
|
||||
row.model = mb_host.model;
|
||||
row.context = mb_host.context;
|
||||
row.compute = mb_host.compute;
|
||||
rows.push_back(std::move(row));
|
||||
const size_t self = mb_host.model + mb_host.context + mb_host.compute;
|
||||
table_data.push_back({
|
||||
template_other,
|
||||
" - Host",
|
||||
"", // total
|
||||
"", // free
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(mb_host.model / MiB),
|
||||
std::to_string(mb_host.context / MiB),
|
||||
std::to_string(mb_host.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "host"},
|
||||
{"name", "Host"},
|
||||
{"self", self / MiB},
|
||||
{"model", mb_host.model / MiB},
|
||||
{"context", mb_host.context / MiB},
|
||||
{"compute", mb_host.compute / MiB},
|
||||
});
|
||||
}
|
||||
|
||||
// all remaining buffer types:
|
||||
// print memory breakdown for all remaining buffer types:
|
||||
for (const auto & buft_mb : memory_breakdown) {
|
||||
ggml_backend_buffer_type_t buft = buft_mb.first;
|
||||
const llama_memory_breakdown_data & mb = buft_mb.second;
|
||||
if (seen_buffer_types.count(buft) == 1) {
|
||||
continue;
|
||||
}
|
||||
common_memory_breakdown_row row;
|
||||
row.kind = common_memory_breakdown_row::BUFFER_TYPE;
|
||||
row.name = ggml_backend_buft_name(buft);
|
||||
row.model = mb.model;
|
||||
row.context = mb.context;
|
||||
row.compute = mb.compute;
|
||||
rows.push_back(std::move(row));
|
||||
|
||||
seen_buffer_types.insert(buft);
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
const std::vector<common_memory_breakdown_row> rows = common_memory_breakdown_rows(ctx);
|
||||
|
||||
std::vector<std::array<std::string, 9>> table_data;
|
||||
table_data.reserve(rows.size() + 1);
|
||||
|
||||
// same data as the table below, for --log-jsonl consumers
|
||||
common_json json_rows = common_json::array();
|
||||
const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
|
||||
const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
|
||||
const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
|
||||
|
||||
table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
|
||||
|
||||
constexpr size_t MiB = 1024 * 1024;
|
||||
|
||||
for (const auto & row : rows) {
|
||||
const size_t self = row.model + row.context + row.compute;
|
||||
|
||||
if (row.kind == common_memory_breakdown_row::DEVICE) {
|
||||
const int64_t unaccounted = row.total - row.free - static_cast<int64_t>(self);
|
||||
|
||||
table_data.push_back({
|
||||
template_gpu,
|
||||
" - " + row.name + " (" + row.description + ")",
|
||||
std::to_string(row.total / MiB),
|
||||
std::to_string(row.free / MiB),
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(row.model / MiB),
|
||||
std::to_string(row.context / MiB),
|
||||
std::to_string(row.compute / MiB),
|
||||
std::to_string(unaccounted / static_cast<int64_t>(MiB))});
|
||||
|
||||
json_rows.push_back({
|
||||
{"kind", "device"},
|
||||
{"name", row.name},
|
||||
{"description", row.description},
|
||||
{"total", row.total / MiB},
|
||||
{"free", row.free / MiB},
|
||||
{"self", self / MiB},
|
||||
{"model", row.model / MiB},
|
||||
{"context", row.context / MiB},
|
||||
{"compute", row.compute / MiB},
|
||||
{"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string name = ggml_backend_buft_name(buft);
|
||||
const size_t self = mb.model + mb.context + mb.compute;
|
||||
table_data.push_back({
|
||||
template_other,
|
||||
" - " + row.name,
|
||||
" - " + name,
|
||||
"", // total
|
||||
"", // free
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(row.model / MiB),
|
||||
std::to_string(row.context / MiB),
|
||||
std::to_string(row.compute / MiB),
|
||||
std::to_string(mb.model / MiB),
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
json_rows.push_back({
|
||||
{"kind", row.kind == common_memory_breakdown_row::HOST ? "host" : "buffer_type"},
|
||||
{"name", row.name},
|
||||
rows.push_back({
|
||||
{"kind", "buffer_type"},
|
||||
{"name", name},
|
||||
{"self", self / MiB},
|
||||
{"model", row.model / MiB},
|
||||
{"context", row.context / MiB},
|
||||
{"compute", row.compute / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
});
|
||||
|
||||
seen_buffer_types.insert(buft);
|
||||
}
|
||||
|
||||
for (size_t j = 1; j < table_data[0].size(); j++) {
|
||||
@@ -1195,21 +1081,20 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
|
||||
LOG_JSON("fit_memory_breakdown", common_json({
|
||||
{"unit", "MiB"},
|
||||
{"rows", json_rows},
|
||||
{"rows", rows},
|
||||
}));
|
||||
}
|
||||
|
||||
void common_fit_print(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams) {
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
GGML_ASSERT(dmd.size() == devs.size() + 1);
|
||||
|
||||
for (size_t id = 0; id < devs.size(); id++) {
|
||||
|
||||
+5
-48
@@ -3,7 +3,6 @@
|
||||
#include "ggml.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
enum common_params_fit_status {
|
||||
@@ -16,7 +15,7 @@ enum common_params_fit_status {
|
||||
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
|
||||
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
|
||||
struct common_fit_extra_model {
|
||||
std::vector<std::string> paths_model;
|
||||
const char * path_model;
|
||||
llama_model_params * mparams;
|
||||
llama_context_params * cparams;
|
||||
bool shares_model;
|
||||
@@ -28,65 +27,24 @@ struct common_fit_extra_model {
|
||||
// - only parameters that have the same value as in llama_default_model_params are modified
|
||||
// with the exception of the context size which is modified if and only if equal to 0
|
||||
common_params_fit_status common_fit_params(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams,
|
||||
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
|
||||
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
|
||||
size_t * margins, // margins of memory to leave per device in bytes
|
||||
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
|
||||
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
|
||||
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
|
||||
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
|
||||
|
||||
// print estimated memory to stdout
|
||||
void common_fit_print(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams);
|
||||
|
||||
void common_memory_breakdown_print(const llama_context * ctx);
|
||||
|
||||
// one row of the memory breakdown: a device, the host or another buffer type
|
||||
struct common_memory_breakdown_row {
|
||||
enum kind_t { DEVICE, HOST, BUFFER_TYPE };
|
||||
|
||||
kind_t kind = DEVICE;
|
||||
std::string name; // device or buffer type name, "Host" for the host row
|
||||
std::string description; // device description, empty otherwise
|
||||
int64_t total = 0; // device memory in bytes, 0 unless kind == DEVICE
|
||||
int64_t free = 0; // free device memory in bytes, 0 unless kind == DEVICE
|
||||
size_t model = 0; // bytes held by this context, by kind of buffer
|
||||
size_t context = 0;
|
||||
size_t compute = 0;
|
||||
};
|
||||
|
||||
// memory a model needs at the min and max context --fit may pick
|
||||
struct common_memory_reports {
|
||||
uint32_t n_ctx_train = 0;
|
||||
uint32_t n_ctx_max = 0; // the largest context that fits the free device memory right now
|
||||
uint32_t n_ctx_min = 0; // --fit-ctx (the smallest context --fit may pick), with the placement set in params
|
||||
std::vector<common_memory_breakdown_row> info_max; // breakdown at n_ctx_max
|
||||
std::vector<common_memory_breakdown_row> info_min; // breakdown at n_ctx_min
|
||||
};
|
||||
|
||||
// rows of common_memory_breakdown_print: model devices first, then host
|
||||
std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llama_context * ctx);
|
||||
|
||||
// load model + context with no_alloc and return the memory breakdown, params are used as-is
|
||||
// n_ctx_train and n_ctx are optional outputs; throws on failure
|
||||
std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
uint32_t * n_ctx_train,
|
||||
uint32_t * n_ctx,
|
||||
ggml_log_level log_level);
|
||||
|
||||
struct common_params;
|
||||
|
||||
// measure min and max without allocating, params may be modified as with --fit; throws on failure
|
||||
common_memory_reports common_measure_memory(common_params & params);
|
||||
|
||||
struct common_device_memory_data {
|
||||
int64_t total;
|
||||
int64_t free;
|
||||
@@ -99,12 +57,11 @@ using common_device_memory_data_vec = std::vector<common_device_memory_data>;
|
||||
|
||||
// Load a model + context with no_alloc and return the per-device memory breakdown.
|
||||
common_device_memory_data_vec common_get_device_memory_data(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level);
|
||||
+13
-12
@@ -45,7 +45,7 @@ static fs::path get_cache_directory() {
|
||||
{HOME_DIR, fs::path(".cache") / "huggingface" / "hub"}
|
||||
};
|
||||
for (const auto & entry : entries) {
|
||||
if (fs::path base = common_get_env(entry.var); !base.empty()) {
|
||||
if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
|
||||
return entry.path.empty() ? base : base / entry.path;
|
||||
}
|
||||
}
|
||||
@@ -63,7 +63,7 @@ static fs::path get_cache_directory() {
|
||||
}
|
||||
|
||||
std::string get_cache_path() {
|
||||
return get_cache_directory().string();
|
||||
return fs_path_to_utf8(get_cache_directory());
|
||||
}
|
||||
|
||||
static std::string folder_name_to_repo(const std::string & folder) {
|
||||
@@ -242,7 +242,7 @@ static std::string get_repo_commit(const std::string & repo_id,
|
||||
LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
|
||||
continue;
|
||||
}
|
||||
const fs::path candidate = _name;
|
||||
const fs::path candidate = fs::u8path(_name);
|
||||
|
||||
if (!is_valid_subpath(refs_path, candidate)) {
|
||||
LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
|
||||
@@ -317,7 +317,7 @@ hf_files get_repo_files(const std::string & repo_id,
|
||||
file.repo_id = repo_id;
|
||||
file.path = item["path"].get<std::string>();
|
||||
|
||||
const fs::path subpath = file.path;
|
||||
const fs::path subpath = fs::u8path(file.path);
|
||||
|
||||
if (!is_valid_subpath(commit_path, subpath)) {
|
||||
LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());
|
||||
@@ -340,11 +340,11 @@ hf_files get_repo_files(const std::string & repo_id,
|
||||
file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
|
||||
|
||||
fs::path final_path = commit_path / subpath;
|
||||
file.final_path = final_path.string();
|
||||
file.final_path = fs_path_to_utf8(final_path);
|
||||
|
||||
if (!file.oid.empty() && !fs::exists(final_path)) {
|
||||
fs::path local_path = blobs_path / file.oid;
|
||||
file.local_path = local_path.string();
|
||||
file.local_path = fs_path_to_utf8(local_path);
|
||||
} else {
|
||||
file.local_path = file.final_path;
|
||||
}
|
||||
@@ -411,7 +411,7 @@ hf_files get_cached_files(const std::string & repo_id) {
|
||||
if (!fs::exists(snapshots_path)) {
|
||||
continue;
|
||||
}
|
||||
std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
|
||||
std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
|
||||
|
||||
if (!is_valid_repo_id(_repo_id)) {
|
||||
continue;
|
||||
@@ -434,8 +434,9 @@ hf_files get_cached_files(const std::string & repo_id) {
|
||||
if (!path.empty()) {
|
||||
hf_file file;
|
||||
file.repo_id = _repo_id;
|
||||
file.path = path.generic_string();
|
||||
file.local_path = entry.path().string();
|
||||
const auto generic_path = path.generic_u8string();
|
||||
file.path = std::string(generic_path.begin(), generic_path.end());
|
||||
file.local_path = fs_path_to_utf8(entry.path());
|
||||
file.final_path = file.local_path;
|
||||
files.push_back(std::move(file));
|
||||
}
|
||||
@@ -449,8 +450,8 @@ std::string finalize_file(const hf_file & file, const std::function<bool()> & ke
|
||||
static std::atomic<bool> symlinks_disabled{false};
|
||||
|
||||
std::error_code ec;
|
||||
fs::path local_path = file.local_path;
|
||||
fs::path final_path = file.final_path;
|
||||
fs::path local_path = fs::u8path(file.local_path);
|
||||
fs::path final_path = fs::u8path(file.final_path);
|
||||
|
||||
if (local_path == final_path || fs::exists(final_path, ec)) {
|
||||
return file.final_path;
|
||||
@@ -515,7 +516,7 @@ bool remove_cached_repo(const std::string & repo_id) {
|
||||
std::error_code ec;
|
||||
auto removed = fs::remove_all(repo_path, ec);
|
||||
if (ec) {
|
||||
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
|
||||
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
|
||||
return false;
|
||||
}
|
||||
return removed > 0;
|
||||
|
||||
@@ -99,26 +99,6 @@ static common_http_url common_http_parse_url(const std::string & url) {
|
||||
return parts;
|
||||
}
|
||||
|
||||
// resolve a redirect Location against the URL it came from
|
||||
static std::string common_http_resolve_location(const std::string & base_url, const std::string & location) {
|
||||
if (location.find("://") != std::string::npos) {
|
||||
return location;
|
||||
}
|
||||
const common_http_url base = common_http_parse_url(base_url);
|
||||
if (location.compare(0, 2, "//") == 0) {
|
||||
return base.scheme + ":" + location;
|
||||
}
|
||||
const std::string origin = base.scheme + "://" + common_http_format_host(base.host) + ":" + std::to_string(base.port);
|
||||
const std::string path = base.path.substr(0, base.path.find('?'));
|
||||
if (!location.empty() && location[0] == '/') {
|
||||
return origin + location;
|
||||
}
|
||||
if (!location.empty() && location[0] == '?') {
|
||||
return origin + path + location;
|
||||
}
|
||||
return origin + path.substr(0, path.rfind('/') + 1) + location;
|
||||
}
|
||||
|
||||
// match a loopback destination: a remote proxy can never reach the client machine's own
|
||||
// loopback, so these hosts are always contacted directly, before NO_PROXY is consulted
|
||||
// (similar to Go's net/http)
|
||||
|
||||
@@ -6,9 +6,7 @@
|
||||
// we can remove it in the future when we figure out a better way to define inputs using jinja::value
|
||||
#include "json.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
|
||||
#define FILENAME "jinja-caps"
|
||||
@@ -186,10 +184,6 @@ caps caps_get(jinja::program & prog) {
|
||||
{
|
||||
{"role", "user"},
|
||||
{"content", json::array({
|
||||
{
|
||||
{"type", "text"},
|
||||
{"text", content_marker}
|
||||
}
|
||||
})}
|
||||
}
|
||||
});
|
||||
@@ -590,68 +584,6 @@ caps caps_get(jinja::program & prog) {
|
||||
}
|
||||
);
|
||||
|
||||
if (result.supports_reasoning_effort) {
|
||||
JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels");
|
||||
|
||||
static const std::vector<std::string> candidates = { "minimal", "low", "medium", "high", "xhigh", "max" };
|
||||
|
||||
// renders a single user turn with the given effort, nothing when the template raises
|
||||
const auto render = [&](const std::string & effort) {
|
||||
std::optional<std::string> rendered;
|
||||
caps_try_execute(
|
||||
prog,
|
||||
[&]() {
|
||||
// messages
|
||||
return json::array({
|
||||
{
|
||||
{"role", "user"},
|
||||
{"content", "User message"}
|
||||
},
|
||||
});
|
||||
},
|
||||
[&](context & ctx) {
|
||||
ctx.set_val("enable_thinking", mk_val<value_bool>(true));
|
||||
caps_apply_reasoning_effort(ctx, effort);
|
||||
},
|
||||
nullptr, // tools_fn
|
||||
[&](context &, bool success, value &, value &, const std::string & output) {
|
||||
if (success) {
|
||||
rendered = output;
|
||||
}
|
||||
}
|
||||
);
|
||||
return rendered;
|
||||
};
|
||||
|
||||
// an unknown level shows the render of the template fallback, a level that renders the same is not honored
|
||||
// levels sharing a render are aliases and the highest one is kept
|
||||
const auto unknown = render("REASONING_EFFORT_UNKNOWN");
|
||||
std::vector<std::string> renders;
|
||||
std::vector<std::string> rejected;
|
||||
for (const auto & level : candidates) {
|
||||
const auto rendered = render(level);
|
||||
if (!rendered) {
|
||||
rejected.push_back(level);
|
||||
continue;
|
||||
}
|
||||
if (rendered == unknown) {
|
||||
continue;
|
||||
}
|
||||
const auto it = std::find(renders.begin(), renders.end(), *rendered);
|
||||
if (it != renders.end()) {
|
||||
result.reasoning_effort_levels[it - renders.begin()] = level;
|
||||
} else {
|
||||
renders.push_back(*rendered);
|
||||
result.reasoning_effort_levels.push_back(level);
|
||||
}
|
||||
}
|
||||
|
||||
// a template that renders no level fails the probe for another reason, so its failures say nothing about the levels
|
||||
if (!result.reasoning_effort_levels.empty()) {
|
||||
result.reasoning_effort_rejected = rejected;
|
||||
}
|
||||
}
|
||||
|
||||
JJ_DEBUG("%s\n", result.to_string().c_str());
|
||||
|
||||
return result;
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
namespace jinja {
|
||||
|
||||
@@ -20,12 +19,6 @@ struct caps {
|
||||
// supports reasoning effort levels
|
||||
bool supports_reasoning_effort = false;
|
||||
|
||||
// reasoning effort levels the template renders distinctly, from lowest to highest
|
||||
std::vector<std::string> reasoning_effort_levels;
|
||||
|
||||
// reasoning effort levels the template raises on
|
||||
std::vector<std::string> reasoning_effort_rejected;
|
||||
|
||||
// one of the 2 content capabilities must be true
|
||||
bool supports_string_content = true;
|
||||
bool supports_typed_content = false;
|
||||
|
||||
@@ -263,6 +263,41 @@ lexer_result lexer::tokenize(const std::string & source) {
|
||||
|
||||
char ch = src[pos];
|
||||
|
||||
bool is_closing_block = ch == '-' && next_pos_is( {'%', '}'} );
|
||||
|
||||
// Check for unary operators
|
||||
if (!is_closing_block && (ch == '-' || ch == '+')) {
|
||||
start_pos = pos;
|
||||
token::type last_token_type = tokens.empty() ? token::eof : tokens.back().t;
|
||||
if (last_token_type == token::text || last_token_type == token::eof) {
|
||||
throw lexer_exception(std::string("unexpected character: ") + ch, source, pos);
|
||||
}
|
||||
switch (last_token_type) {
|
||||
case token::identifier:
|
||||
case token::numeric_literal:
|
||||
case token::string_literal:
|
||||
case token::close_paren:
|
||||
case token::close_square_bracket:
|
||||
// Part of a binary operator
|
||||
// a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1
|
||||
// Continue parsing normally
|
||||
break;
|
||||
default: {
|
||||
// Is part of a unary operator
|
||||
// (-1), [-1], (1 + -1), not -1, -apple
|
||||
++pos; // Consume the operator
|
||||
|
||||
// Check for numbers following the unary operator
|
||||
std::string num = consume_numeric();
|
||||
std::string value = std::string(1, ch) + num;
|
||||
token::type t = num.empty() ? token::unary_operator : token::numeric_literal;
|
||||
// JJ_DEBUG("consumed unary operator or numeric literal: '%s'", value.c_str());
|
||||
tokens.push_back({t, value, start_pos});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to match one of the tokens in the mapping table
|
||||
bool matched = false;
|
||||
for (const auto & [seq, typ] : ordered_mapping_table) {
|
||||
|
||||
@@ -37,8 +37,8 @@ struct token {
|
||||
call_operator, // ()
|
||||
additive_binary_operator, // + - ~
|
||||
multiplicative_binary_operator, // * / %
|
||||
power_operator, // **
|
||||
comparison_binary_operator, // < > <= >= == !=
|
||||
unary_operator, // ! - +
|
||||
comment, // {# ... #}
|
||||
};
|
||||
type t;
|
||||
@@ -71,8 +71,8 @@ static std::string type_to_string(token::type t) {
|
||||
case token::call_operator: return "call_operator";
|
||||
case token::additive_binary_operator: return "additive_binary_operator";
|
||||
case token::multiplicative_binary_operator: return "multiplicative_binary_operator";
|
||||
case token::power_operator: return "power_operator";
|
||||
case token::comparison_binary_operator: return "comparison_binary_operator";
|
||||
case token::unary_operator: return "unary_operator";
|
||||
case token::comment: return "comment";
|
||||
default: return "unknown";
|
||||
}
|
||||
@@ -137,9 +137,7 @@ struct lexer {
|
||||
{"+", token::additive_binary_operator},
|
||||
{"-", token::additive_binary_operator},
|
||||
{"~", token::additive_binary_operator},
|
||||
{"**", token::power_operator},
|
||||
{"*", token::multiplicative_binary_operator},
|
||||
{"//", token::multiplicative_binary_operator},
|
||||
{"/", token::multiplicative_binary_operator},
|
||||
{"%", token::multiplicative_binary_operator},
|
||||
// Assignment operator
|
||||
|
||||
+16
-50
@@ -21,17 +21,6 @@ static bool is_type(const statement_ptr & ptr) {
|
||||
class parser {
|
||||
const std::vector<token> & tokens;
|
||||
size_t current = 0;
|
||||
int depth = 0;
|
||||
|
||||
struct depth_guard {
|
||||
parser & p;
|
||||
depth_guard(parser & p) : p(p) {
|
||||
if (++p.depth > JINJA_RECURSIVE_LIMIT) {
|
||||
throw parser_exception("Parser Error: template is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep", p.source, p.peek().pos);
|
||||
}
|
||||
}
|
||||
~depth_guard() { p.depth--; }
|
||||
};
|
||||
|
||||
std::string source; // for error reporting
|
||||
|
||||
@@ -105,7 +94,6 @@ private:
|
||||
}
|
||||
|
||||
statement_ptr parse_any() {
|
||||
depth_guard guard(*this);
|
||||
size_t start_pos = current;
|
||||
switch (peek().t) {
|
||||
case token::comment:
|
||||
@@ -178,7 +166,7 @@ private:
|
||||
// bool has_caller_args = false;
|
||||
if (is(token::open_paren)) {
|
||||
// Optional caller arguments, e.g. {% call(user) dump_users(...) %}
|
||||
caller_args = parse_args(false);
|
||||
caller_args = parse_args();
|
||||
// has_caller_args = true;
|
||||
}
|
||||
auto callee = parse_primary_expression();
|
||||
@@ -266,7 +254,6 @@ private:
|
||||
size_t pos0 = current;
|
||||
++current; // consume {%
|
||||
++current; // consume 'elif'
|
||||
depth_guard guard(*this);
|
||||
alternate.push_back(parse_if_statement(pos0)); // nested If
|
||||
} else if (is_statement({"else"})) {
|
||||
++current; // consume {%
|
||||
@@ -283,7 +270,7 @@ private:
|
||||
|
||||
statement_ptr parse_macro_statement(size_t start_pos) {
|
||||
auto name = parse_primary_expression();
|
||||
auto args = parse_args(false);
|
||||
auto args = parse_args();
|
||||
expect(token::close_statement, "Expected %}");
|
||||
statements body;
|
||||
// Keep going until we hit {% endmacro
|
||||
@@ -343,7 +330,6 @@ private:
|
||||
}
|
||||
|
||||
statement_ptr parse_if_expression() {
|
||||
depth_guard guard(*this);
|
||||
auto a = parse_logical_or_expression();
|
||||
if (is_identifier("if")) {
|
||||
// Ternary expression
|
||||
@@ -387,7 +373,6 @@ private:
|
||||
statement_ptr parse_logical_negation_expression() {
|
||||
// Try parse unary operators
|
||||
if (is_identifier("not")) {
|
||||
depth_guard guard(*this);
|
||||
size_t start_pos = current;
|
||||
auto op = next();
|
||||
return mk_stmt<unary_expression>(start_pos, op, parse_logical_negation_expression());
|
||||
@@ -397,13 +382,11 @@ private:
|
||||
|
||||
statement_ptr parse_comparison_expression() {
|
||||
// NOTE: membership has same precedence as comparison
|
||||
// e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in 'apple' and 'apple' == 'b' and 'b' in 'banana')
|
||||
statements operands;
|
||||
std::vector<token> ops;
|
||||
operands.push_back(parse_additive_expression());
|
||||
size_t pos = current;
|
||||
// e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana')))
|
||||
auto left = parse_additive_expression();
|
||||
while (true) {
|
||||
token op;
|
||||
size_t start_pos = current;
|
||||
if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") {
|
||||
op = {token::identifier, "not in", tokens[current].pos};
|
||||
++current; // consume 'not'
|
||||
@@ -413,17 +396,9 @@ private:
|
||||
} else if (is(token::comparison_binary_operator)) {
|
||||
op = next();
|
||||
} else break;
|
||||
ops.push_back(std::move(op));
|
||||
operands.push_back(parse_additive_expression());
|
||||
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_additive_expression());
|
||||
}
|
||||
if (ops.empty()) {
|
||||
return std::move(operands[0]);
|
||||
}
|
||||
if (ops.size() == 1) {
|
||||
return mk_stmt<binary_expression>(pos, ops[0], std::move(operands[0]), std::move(operands[1]));
|
||||
}
|
||||
// chained comparison, e.g. 1 <= x <= 100
|
||||
return mk_stmt<comparison_chain_expression>(pos, std::move(operands), std::move(ops));
|
||||
return left;
|
||||
}
|
||||
|
||||
statement_ptr parse_additive_expression() {
|
||||
@@ -482,9 +457,7 @@ private:
|
||||
}
|
||||
|
||||
statement_ptr parse_unary_expression() {
|
||||
// a sign where an operand is expected is unary: -x, 1 + -x, x if c else -1
|
||||
if (is(token::additive_binary_operator) && peek().value != "~") {
|
||||
depth_guard guard(*this);
|
||||
if (is(token::unary_operator)) {
|
||||
size_t start_pos = current;
|
||||
auto op = next();
|
||||
return mk_stmt<unary_expression>(start_pos, op, parse_unary_expression());
|
||||
@@ -504,29 +477,22 @@ private:
|
||||
size_t start_pos = current;
|
||||
auto expr = mk_stmt<call_expression>(start_pos, std::move(callee), parse_args());
|
||||
auto member = parse_member_expression(std::move(expr)); // foo.x().y
|
||||
if (!is(token::open_paren)) {
|
||||
return member;
|
||||
}
|
||||
depth_guard guard(*this);
|
||||
return parse_call_expression(std::move(member)); // foo.x()()
|
||||
return is(token::open_paren)
|
||||
? parse_call_expression(std::move(member)) // foo.x()()
|
||||
: std::move(member);
|
||||
}
|
||||
|
||||
statements parse_args(bool allow_spread = true) {
|
||||
statements parse_args() {
|
||||
// comma-separated arguments list
|
||||
expect(token::open_paren, "Expected (");
|
||||
statements args;
|
||||
while (!is(token::close_paren)) {
|
||||
statement_ptr arg;
|
||||
// unpacking: *expr or **expr
|
||||
const auto & t = peek();
|
||||
if (t.t == token::power_operator || (t.t == token::multiplicative_binary_operator && t.value == "*")) {
|
||||
if (!allow_spread) {
|
||||
throw std::runtime_error("Unexpected token: " + t.value + " of type " + std::to_string(t.t));
|
||||
}
|
||||
// unpacking: *expr
|
||||
if (peek().t == token::multiplicative_binary_operator && peek().value == "*") {
|
||||
size_t start_pos = current;
|
||||
++current; // consume * or **
|
||||
bool is_kwargs = t.t == token::power_operator;
|
||||
arg = mk_stmt<spread_expression>(start_pos, parse_expression(), is_kwargs);
|
||||
++current; // consume *
|
||||
arg = mk_stmt<spread_expression>(start_pos, parse_expression());
|
||||
} else {
|
||||
arg = parse_expression();
|
||||
if (is(token::equals)) {
|
||||
|
||||
+128
-180
@@ -3,7 +3,6 @@
|
||||
#include "value.h"
|
||||
#include "utils.h"
|
||||
|
||||
#include <exception>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
@@ -53,58 +52,50 @@ static void ensure_key_type_allowed(const value & val) {
|
||||
|
||||
// execute with error handling
|
||||
value statement::execute(context & ctx) const {
|
||||
std::exception_ptr pending;
|
||||
std::string message;
|
||||
try {
|
||||
return execute_impl(ctx);
|
||||
} catch (const continue_statement::signal & /* ex */) {
|
||||
pending = std::current_exception();
|
||||
throw;
|
||||
} catch (const break_statement::signal & /* ex */) {
|
||||
pending = std::current_exception();
|
||||
throw;
|
||||
} catch (const rethrown_exception & /* ex */) {
|
||||
pending = std::current_exception();
|
||||
throw;
|
||||
} catch (const not_implemented_exception & /* ex */) {
|
||||
pending = std::current_exception();
|
||||
throw;
|
||||
} catch (const std::exception & e) {
|
||||
const std::string & source = *ctx.src;
|
||||
if (source.empty()) {
|
||||
std::ostringstream oss;
|
||||
oss << "\nError executing " << type() << " at position " << pos << ": " << e.what();
|
||||
message = oss.str();
|
||||
throw rethrown_exception(oss.str());
|
||||
} else {
|
||||
std::ostringstream oss;
|
||||
oss << "\n------------\n";
|
||||
oss << "While executing " << type() << " at " << get_line_col(source, pos) << " in source:\n";
|
||||
oss << peak_source(source, pos) << "\n";
|
||||
oss << "Error: " << e.what();
|
||||
message = oss.str();
|
||||
// throw as another exception to avoid repeated formatting
|
||||
throw rethrown_exception(oss.str());
|
||||
}
|
||||
}
|
||||
// throw outside the catch: MSVC keeps the unwound frames on the stack while a catch block runs
|
||||
if (pending) {
|
||||
std::rethrow_exception(pending);
|
||||
}
|
||||
// throw as another exception to avoid repeated formatting
|
||||
throw rethrown_exception(message);
|
||||
}
|
||||
|
||||
value identifier::execute_impl(context & ctx) const {
|
||||
auto it = ctx.find_val(key);
|
||||
if (it && !it->is_undefined()) {
|
||||
auto it = ctx.get_val(val);
|
||||
auto builtins = global_builtins();
|
||||
if (!it->is_undefined()) {
|
||||
if (ctx.is_get_stats) {
|
||||
value_t::stats_t::mark_used(it);
|
||||
}
|
||||
JJ_DEBUG("Identifier '%s' found, type = %s", val.c_str(), it->type().c_str());
|
||||
return it;
|
||||
}
|
||||
const auto & builtins = global_builtins();
|
||||
auto builtin = builtins.find(val);
|
||||
if (builtin != builtins.end()) {
|
||||
} else if (builtins.find(val) != builtins.end()) {
|
||||
JJ_DEBUG("Identifier '%s' found in builtins", val.c_str());
|
||||
return mk_val<value_func>(val, builtin->second);
|
||||
return mk_val<value_func>(val, builtins.at(val));
|
||||
} else {
|
||||
JJ_DEBUG("Identifier '%s' not found, returning undefined", val.c_str());
|
||||
return mk_val<value_undefined>(val);
|
||||
}
|
||||
JJ_DEBUG("Identifier '%s' not found, returning undefined", val.c_str());
|
||||
return mk_val<value_undefined>(val);
|
||||
}
|
||||
|
||||
value object_literal::execute_impl(context & ctx) const {
|
||||
@@ -118,8 +109,20 @@ value object_literal::execute_impl(context & ctx) const {
|
||||
return obj;
|
||||
}
|
||||
|
||||
static value exec_binary_op(context & ctx, const token & op, value left_val, value right_val) {
|
||||
value binary_expression::execute_impl(context & ctx) const {
|
||||
value left_val = left->execute(ctx);
|
||||
|
||||
// Logical operators
|
||||
if (op.value == "and") {
|
||||
JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? right->execute(ctx) : std::move(left_val);
|
||||
} else if (op.value == "or") {
|
||||
JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? std::move(left_val) : right->execute(ctx);
|
||||
}
|
||||
|
||||
// Equality operators
|
||||
value right_val = right->execute(ctx);
|
||||
JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right_val->type().c_str());
|
||||
if (op.value == "==") {
|
||||
return mk_val<value_bool>(*left_val == *right_val);
|
||||
@@ -191,30 +194,11 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(res);
|
||||
} else {
|
||||
return mk_val<value_int>(float_to_int(res));
|
||||
return mk_val<value_int>(static_cast<int64_t>(res));
|
||||
}
|
||||
} else if (op.value == "/") {
|
||||
JJ_DEBUG("Division operation: %f / %f", a, b);
|
||||
return mk_val<value_float>(a / b);
|
||||
} else if (op.value == "//") {
|
||||
// python semantics: round toward -inf
|
||||
if (is_val<value_int>(left_val) && is_val<value_int>(right_val)) {
|
||||
const int64_t x = left_val->as_int();
|
||||
const int64_t y = right_val->as_int();
|
||||
if (y == 0) {
|
||||
throw std::runtime_error("integer division by zero");
|
||||
}
|
||||
if (y == -1) {
|
||||
// INT64_MIN / -1 is UB, wrap around instead
|
||||
return mk_val<value_int>((int64_t) (0 - (uint64_t) x));
|
||||
}
|
||||
int64_t q = x / y;
|
||||
if (x % y != 0 && (x < 0) != (y < 0)) {
|
||||
q -= 1;
|
||||
}
|
||||
return mk_val<value_int>(q);
|
||||
}
|
||||
return mk_val<value_float>(std::floor(a / b));
|
||||
} else if (op.value == "%") {
|
||||
double rem = std::fmod(a, b);
|
||||
JJ_DEBUG("Modulo operation: %f %% %f = %f", a, b, rem);
|
||||
@@ -222,7 +206,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(rem);
|
||||
} else {
|
||||
return mk_val<value_int>(float_to_int(rem));
|
||||
return mk_val<value_int>(static_cast<int64_t>(rem));
|
||||
}
|
||||
} else if (op.value == "<") {
|
||||
JJ_DEBUG("Comparison operation: %f < %f is %d", a, b, a < b);
|
||||
@@ -293,17 +277,12 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
// String membership
|
||||
if (is_val<value_string>(left_val) && is_val<value_string>(right_val)) {
|
||||
// case: "a" in "abc"
|
||||
if (op.value == "in" || op.value == "not in") {
|
||||
bool member = test_is_in();
|
||||
return mk_val<value_bool>(op.value == "in" ? member : !member);
|
||||
bool member = test_is_in();
|
||||
if (op.value == "in") {
|
||||
return mk_val<value_bool>(member);
|
||||
} else if (op.value == "not in") {
|
||||
return mk_val<value_bool>(!member);
|
||||
}
|
||||
// case: "a" < "b"; UTF-8 byte order is the same as codepoint order, like python
|
||||
const std::string a = left_val->as_string().str();
|
||||
const std::string b = right_val->as_string().str();
|
||||
if (op.value == "<") { return mk_val<value_bool>(a < b); }
|
||||
if (op.value == ">") { return mk_val<value_bool>(a > b); }
|
||||
if (op.value == "<=") { return mk_val<value_bool>(a <= b); }
|
||||
if (op.value == ">=") { return mk_val<value_bool>(a >= b); }
|
||||
}
|
||||
|
||||
// Value key in object
|
||||
@@ -320,74 +299,58 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
|
||||
}
|
||||
|
||||
value binary_expression::execute_impl(context & ctx) const {
|
||||
value left_val = left->execute(ctx);
|
||||
|
||||
// Logical operators
|
||||
if (op.value == "and") {
|
||||
JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? right->execute(ctx) : std::move(left_val);
|
||||
} else if (op.value == "or") {
|
||||
JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str());
|
||||
return left_val->as_bool() ? std::move(left_val) : right->execute(ctx);
|
||||
static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
|
||||
JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
|
||||
if (ctx.is_get_stats) {
|
||||
value_t::stats_t::mark_used(input);
|
||||
input->stats.ops.insert(name);
|
||||
}
|
||||
|
||||
return exec_binary_op(ctx, op, std::move(left_val), right->execute(ctx));
|
||||
}
|
||||
|
||||
value comparison_chain_expression::execute_impl(context & ctx) const {
|
||||
value left_val = operands[0]->execute(ctx);
|
||||
value res;
|
||||
for (size_t i = 0; i < ops.size(); ++i) {
|
||||
value right_val = operands[i + 1]->execute(ctx);
|
||||
res = exec_binary_op(ctx, ops[i], left_val, right_val);
|
||||
if (!res->as_bool()) {
|
||||
break; // same as "and", the next operands are not executed
|
||||
}
|
||||
left_val = std::move(right_val);
|
||||
auto builtins = input->get_builtins();
|
||||
auto it = builtins.find(name);
|
||||
if (it != builtins.end()) {
|
||||
JJ_DEBUG("Binding built-in '%s'", name.c_str());
|
||||
return mk_val<value_func>(name, it->second, input);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
static void exec_call_args(context & ctx, const statements & arg_stmts, func_args & args) {
|
||||
for (const auto & arg_stmt : arg_stmts) {
|
||||
if (!is_stmt<spread_expression>(arg_stmt)) {
|
||||
auto arg_val = arg_stmt->execute(ctx);
|
||||
JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
|
||||
args.push_back(arg_val);
|
||||
continue;
|
||||
}
|
||||
auto spread = cast_stmt<spread_expression>(arg_stmt);
|
||||
value val = spread->argument->execute(ctx);
|
||||
JJ_DEBUG(" Unpacked argument type: %s", val->type().c_str());
|
||||
if (spread->is_kwargs) {
|
||||
if (!is_val<value_object>(val)) {
|
||||
throw std::runtime_error("Argument after ** must be an object, got " + val->type());
|
||||
}
|
||||
for (const auto & pair : val->as_ordered_object()) {
|
||||
if (!is_val<value_string>(pair.first)) {
|
||||
throw std::runtime_error("Keyword argument key must be a string, got " + pair.first->type());
|
||||
}
|
||||
args.push_back(mk_val<value_kwarg>(pair.first->as_string().str(), pair.second));
|
||||
}
|
||||
} else {
|
||||
if (!is_val<value_array>(val)) {
|
||||
throw std::runtime_error("Argument after * must be an array, got " + val->type());
|
||||
}
|
||||
for (const auto & item : val->as_array()) {
|
||||
args.push_back(item);
|
||||
}
|
||||
}
|
||||
if (undef_on_missing) {
|
||||
return mk_val<value_undefined>(name);
|
||||
}
|
||||
throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
|
||||
}
|
||||
|
||||
static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
|
||||
JJ_DEBUG("Applying filter to %s", input->type().c_str());
|
||||
|
||||
auto set_filter_alias = [](auto & filter_id) {
|
||||
if (filter_id == "count") {
|
||||
filter_id = "length";
|
||||
} else if (filter_id == "d") {
|
||||
filter_id = "default";
|
||||
} else if (filter_id == "e") {
|
||||
filter_id = "escape";
|
||||
} else if (filter_id == "trim") {
|
||||
filter_id = "strip";
|
||||
}
|
||||
};
|
||||
|
||||
if (is_stmt<identifier>(filter)) {
|
||||
auto filter_id = cast_stmt<identifier>(filter)->val;
|
||||
|
||||
set_filter_alias(filter_id);
|
||||
JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
|
||||
return apply_filter_by_name(ctx, filter_id, input, func_args(ctx));
|
||||
// TODO: Refactor filters so this coercion can be done automatically
|
||||
if (!input->is_undefined() && !is_val<value_string>(input) && (
|
||||
filter_id == "capitalize" ||
|
||||
filter_id == "lower" ||
|
||||
filter_id == "replace" ||
|
||||
filter_id == "strip" ||
|
||||
filter_id == "title" ||
|
||||
filter_id == "upper" ||
|
||||
filter_id == "wordcount"
|
||||
)) {
|
||||
JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
|
||||
input = mk_val<value_string>(input->as_string());
|
||||
}
|
||||
return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx));
|
||||
|
||||
} else if (is_stmt<call_expression>(filter)) {
|
||||
auto call = cast_stmt<call_expression>(filter);
|
||||
@@ -396,11 +359,14 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
|
||||
}
|
||||
auto filter_id = cast_stmt<identifier>(call->callee)->val;
|
||||
|
||||
set_filter_alias(filter_id);
|
||||
JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
|
||||
func_args args(ctx);
|
||||
exec_call_args(ctx, call->args, args);
|
||||
for (const auto & arg_expr : call->args) {
|
||||
args.push_back(arg_expr->execute(ctx));
|
||||
}
|
||||
|
||||
return apply_filter_by_name(ctx, filter_id, input, args);
|
||||
return try_builtin_func(ctx, filter_id, input)->invoke(args);
|
||||
|
||||
} else {
|
||||
throw std::runtime_error("Invalid filter expression");
|
||||
@@ -441,7 +407,9 @@ value test_expression::execute_impl(context & ctx) const {
|
||||
test_id = cast_stmt<identifier>(call->callee)->val;
|
||||
|
||||
JJ_DEBUG("Applying test '%s' with arguments to %s", test_id.c_str(), input->type().c_str());
|
||||
exec_call_args(ctx, call->args, args);
|
||||
for (const auto & arg_expr : call->args) {
|
||||
args.push_back(arg_expr->execute(ctx));
|
||||
}
|
||||
|
||||
} else {
|
||||
throw std::runtime_error("Invalid test expression");
|
||||
@@ -718,7 +686,7 @@ value set_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto obj_ptr = cast_val<value_object>(object);
|
||||
JJ_DEBUG("Setting object property '%s' with value type %s", prop_name.c_str(), rhs->type().c_str());
|
||||
obj_ptr->insert(cached_val<value_string>(ctx, member->static_property), rhs);
|
||||
obj_ptr->insert(prop_name, rhs);
|
||||
|
||||
} else {
|
||||
throw std::runtime_error("Invalid LHS inside assignment expression: " + assignee->type());
|
||||
@@ -779,11 +747,7 @@ value macro_statement::execute_impl(context & ctx) const {
|
||||
std::string name = cast_stmt<identifier>(this->name)->val;
|
||||
|
||||
const func_handler func = [this, name](const func_args & args) -> value {
|
||||
if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
context macro_ctx(args.ctx); // new scope for macro execution
|
||||
macro_ctx.macro_depth = args.ctx.macro_depth + 1;
|
||||
|
||||
bind_parameters(name, this->args, args, macro_ctx);
|
||||
|
||||
@@ -811,15 +775,10 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto * callee_func = cast_val<value_func>(callee_val);
|
||||
|
||||
// caller() can be saved and called after this scope ended, it owns a snapshot of the variables
|
||||
auto caller_ctx = std::make_shared<context>(ctx.snapshot());
|
||||
context caller_ctx(ctx); // new scope for caller execution
|
||||
|
||||
const func_handler func = [this, caller_ctx](const func_args & args) -> value {
|
||||
if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
context block_ctx(*caller_ctx); // new scope for block execution
|
||||
block_ctx.macro_depth = args.ctx.macro_depth + 1;
|
||||
const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
|
||||
context block_ctx(caller_ctx); // new scope for block execution
|
||||
|
||||
bind_parameters("caller", this->caller_args, args, block_ctx);
|
||||
|
||||
@@ -833,25 +792,17 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
call_ctx.set_val("caller", mk_val<value_func>("caller", func));
|
||||
|
||||
func_args args(call_ctx);
|
||||
exec_call_args(ctx, call_expr->args, args);
|
||||
|
||||
for (const auto & arg_expr : call_expr->args) {
|
||||
auto arg_val = arg_expr->execute(ctx);
|
||||
JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
|
||||
args.push_back(arg_val);
|
||||
}
|
||||
|
||||
JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count());
|
||||
return callee_func->invoke(args);
|
||||
}
|
||||
|
||||
member_expression::member_expression(statement_ptr && object, statement_ptr && property, bool computed) :
|
||||
object(std::move(object)),
|
||||
property(std::move(property)),
|
||||
computed(computed) {
|
||||
chk_type<expression>(this->object);
|
||||
chk_type<expression>(this->property);
|
||||
if (!computed) {
|
||||
if (const auto * id = cast_stmt<identifier>(this->property)) {
|
||||
static_property = id->key;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
value member_expression::execute_impl(context & ctx) const {
|
||||
value object = this->object->execute(ctx);
|
||||
|
||||
@@ -860,13 +811,14 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
// syntax: obj[expr]
|
||||
JJ_DEBUG("Member expression, computing property type %s", this->property->type().c_str());
|
||||
|
||||
int64_t arr_size = 0;
|
||||
if (is_val<value_array>(object)) {
|
||||
arr_size = object->as_array().size();
|
||||
} else if (is_val<value_string>(object)) {
|
||||
arr_size = object->as_string().length();
|
||||
}
|
||||
|
||||
if (is_stmt<slice_expression>(this->property)) {
|
||||
int64_t arr_size = 0;
|
||||
if (is_val<value_array>(object)) {
|
||||
arr_size = static_cast<int64_t>(object->as_array().size());
|
||||
} else if (is_val<value_string>(object)) {
|
||||
arr_size = static_cast<int64_t>(cast_val<value_string>(object)->val_str.length());
|
||||
}
|
||||
auto s = cast_stmt<slice_expression>(this->property);
|
||||
value step_val = s->step_expr ? s->step_expr->execute(ctx) : mk_val<value_int>(1);
|
||||
value start_val = s->start_expr ? s->start_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val<value_int>(arr_size - 1) : mk_val<value_int>(0));
|
||||
@@ -888,7 +840,7 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
}
|
||||
} else if (is_stmt<integer_literal>(this->property)) {
|
||||
// syntax: obj.index
|
||||
property = this->property->execute(ctx);
|
||||
property = mk_val<value_int>(cast_stmt<integer_literal>(this->property)->val);
|
||||
if (property->as_int() < 0) {
|
||||
throw std::runtime_error("Static member property cannot be negative");
|
||||
}
|
||||
@@ -897,8 +849,8 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
if (!is_stmt<identifier>(this->property)) {
|
||||
throw std::runtime_error("Static member property must be an identifier");
|
||||
}
|
||||
property = cached_val<value_string>(ctx, static_property);
|
||||
const auto & prop = cast_stmt<identifier>(this->property)->val;
|
||||
property = mk_val<value_string>(cast_stmt<identifier>(this->property)->val);
|
||||
std::string prop = property->as_string().str();
|
||||
JJ_DEBUG("Member expression, object type %s, static property '%s'", object->type().c_str(), prop.c_str());
|
||||
|
||||
// behavior of jinja2: obj having prop as a built-in function AND 'prop', as an object key,
|
||||
@@ -906,56 +858,51 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
// while obj['prop'] returns the property value.
|
||||
// example: {"obj": {"items": 123}} -> obj.items is the built-in function, obj['items'] is 123
|
||||
|
||||
value val = find_builtin_func(ctx, prop, object);
|
||||
if (val) {
|
||||
value val = try_builtin_func(ctx, prop, object, true);
|
||||
if (!is_val<value_undefined>(val)) {
|
||||
return val;
|
||||
}
|
||||
// else, fallthrough to normal property access below
|
||||
}
|
||||
|
||||
JJ_DEBUG("Member expression on object type %s, property type %s", object->type().c_str(), property->type().c_str());
|
||||
value val;
|
||||
value val = mk_val<value_undefined>("object_property");
|
||||
|
||||
if (property->is_undefined()) {
|
||||
JJ_DEBUG("%s", "Member expression property is undefined, returning undefined");
|
||||
return mk_val<value_undefined>("object_property");
|
||||
return val;
|
||||
}
|
||||
|
||||
ensure_key_type_allowed(property);
|
||||
|
||||
if (is_val<value_undefined>(object)) {
|
||||
JJ_DEBUG("%s", "Accessing property on undefined object, returning undefined");
|
||||
return mk_val<value_undefined>("object_property");
|
||||
return val;
|
||||
|
||||
} else if (is_val<value_object>(object)) {
|
||||
val = object->find(property);
|
||||
if (!val || is_val<value_undefined>(val)) {
|
||||
val = try_builtin_func(ctx, property->as_string().str(), object, true);
|
||||
auto key = property->as_string().str();
|
||||
val = object->at(property, val);
|
||||
if (is_val<value_undefined>(val)) {
|
||||
val = try_builtin_func(ctx, key, object, true);
|
||||
}
|
||||
JJ_DEBUG("Accessed property '%s' value, got type: %s", property->as_string().str().c_str(), val->type().c_str());
|
||||
JJ_DEBUG("Accessed property '%s' value, got type: %s", key.c_str(), val->type().c_str());
|
||||
|
||||
} else if (is_val<value_array>(object) || is_val<value_string>(object)) {
|
||||
if (is_val<value_int>(property) || is_val<value_bool>(property)) {
|
||||
int64_t index = property->as_int();
|
||||
JJ_DEBUG("Accessing %s index %d", object->type().c_str(), (int)index);
|
||||
if (is_val<value_array>(object)) {
|
||||
const auto * arr = cast_val<value_array>(object);
|
||||
const auto count = arr->as_array().size();
|
||||
auto & arr = object->as_array();
|
||||
if (index < 0) {
|
||||
// Avoid negating INT64_MIN.
|
||||
const uint64_t distance = static_cast<uint64_t>(-(index + 1)) + 1;
|
||||
if (distance <= count) {
|
||||
val = arr->as_array().at(static_cast<size_t>(count - distance));
|
||||
}
|
||||
} else if (static_cast<uint64_t>(index) < count) {
|
||||
val = arr->as_array().at(static_cast<size_t>(index));
|
||||
index += static_cast<int64_t>(arr.size());
|
||||
}
|
||||
if (index >= 0 && index < static_cast<int64_t>(arr.size())) {
|
||||
val = arr[index];
|
||||
}
|
||||
} else { // value_string
|
||||
if (index >= 0) {
|
||||
const auto byte = cast_val<value_string>(object)->val_str.byte_at(static_cast<size_t>(index));
|
||||
if (byte) {
|
||||
val = mk_val<value_string>(std::string(1, *byte));
|
||||
}
|
||||
auto str = object->as_string().str();
|
||||
if (index >= 0 && index < static_cast<int64_t>(str.size())) {
|
||||
val = mk_val<value_string>(std::string(1, str[index]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -973,10 +920,7 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
val = try_builtin_func(ctx, key, object, true);
|
||||
}
|
||||
|
||||
if (!val) {
|
||||
val = mk_val<value_undefined>("object_property");
|
||||
}
|
||||
if (ctx.is_get_stats && object && property) {
|
||||
if (ctx.is_get_stats && val && object && property) {
|
||||
value_t::stats_t::mark_used(val);
|
||||
value_t::stats_t::mark_used(object);
|
||||
value_t::stats_t::mark_used(property);
|
||||
@@ -993,7 +937,11 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
value call_expression::execute_impl(context & ctx) const {
|
||||
// gather arguments
|
||||
func_args args(ctx);
|
||||
exec_call_args(ctx, this->args, args);
|
||||
for (auto & arg_stmt : this->args) {
|
||||
auto arg_val = arg_stmt->execute(ctx);
|
||||
JJ_DEBUG(" Argument type: %s", arg_val->type().c_str());
|
||||
args.push_back(arg_val);
|
||||
}
|
||||
// execute callee
|
||||
value callee_val = callee->execute(ctx);
|
||||
if (!is_val<value_func>(callee_val)) {
|
||||
|
||||
+23
-91
@@ -58,8 +58,6 @@ struct context {
|
||||
|
||||
bool is_get_stats = false; // whether to collect stats
|
||||
|
||||
int macro_depth = 0;
|
||||
|
||||
visitor_fn visitor;
|
||||
|
||||
// src is optional, used for error reporting
|
||||
@@ -80,44 +78,12 @@ struct context {
|
||||
src(parent.src),
|
||||
current_time(parent.current_time),
|
||||
is_get_stats(parent.is_get_stats),
|
||||
macro_depth(parent.macro_depth),
|
||||
// child scope: own variables only, lookups fall back to the parent (which must outlive this scope)
|
||||
parent(&parent),
|
||||
env(mk_val<value_object>()) {
|
||||
env->has_builtins = false;
|
||||
}
|
||||
|
||||
context(context &&) = default;
|
||||
|
||||
// copy with all visible variables flattened in, for a scope that can outlive its parents (caller() of a call block)
|
||||
context snapshot() const {
|
||||
std::vector<const context *> chain;
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
chain.push_back(c);
|
||||
}
|
||||
context res(*this);
|
||||
res.parent = nullptr;
|
||||
for (auto it = chain.rbegin(); it != chain.rend(); ++it) {
|
||||
for (const auto & kv : (*it)->env->as_ordered_object()) {
|
||||
res.env->insert(kv.first, kv.second);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// returns nullptr if not found
|
||||
value find_val(const value & key) {
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
if (value v = c->env->find(key)) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
// inherit variables (for example, when entering a new scope)
|
||||
env(mk_val<value_object>(*parent.env)) {}
|
||||
|
||||
value get_val(const std::string & name) {
|
||||
auto result = find_val(mk_val<value_string>(name));
|
||||
return result ? result : mk_val<value_undefined>(name);
|
||||
value default_val = mk_val<value_undefined>(name);
|
||||
return env->at(name, default_val);
|
||||
}
|
||||
|
||||
void set_val(const std::string & name, const value & val) {
|
||||
@@ -133,16 +99,9 @@ struct context {
|
||||
}
|
||||
|
||||
private:
|
||||
const context * parent = nullptr;
|
||||
value_object env;
|
||||
};
|
||||
|
||||
// values cached in the AST are shared across renders, but stats mutate values, so stats mode gets a fresh copy
|
||||
template<typename T>
|
||||
value cached_val(const context & ctx, const value & cached) {
|
||||
return ctx.is_get_stats ? mk_val<T>(*cast_val<T>(cached)) : cached;
|
||||
}
|
||||
|
||||
// utils for visiting AST nodes
|
||||
static std::vector<const statement *> stmts_to_ptr(const statements & stmts) {
|
||||
std::vector<const statement *> children;
|
||||
@@ -366,8 +325,11 @@ struct member_expression : public expression {
|
||||
statement_ptr property;
|
||||
bool computed; // true if obj[expr] and false if obj.prop
|
||||
|
||||
value static_property; // set if property is an identifier, see cached_val()
|
||||
member_expression(statement_ptr && object, statement_ptr && property, bool computed);
|
||||
member_expression(statement_ptr && object, statement_ptr && property, bool computed)
|
||||
: object(std::move(object)), property(std::move(property)), computed(computed) {
|
||||
chk_type<expression>(this->object);
|
||||
chk_type<expression>(this->property);
|
||||
}
|
||||
std::string type() const override { return "MemberExpression"; }
|
||||
value execute_impl(context & ctx) const override;
|
||||
void visit(context & ctx) const override {
|
||||
@@ -402,8 +364,7 @@ struct call_expression : public expression {
|
||||
*/
|
||||
struct identifier : public expression {
|
||||
std::string val;
|
||||
const value key; // lookup key, safe to share across renders as lookups do not touch stats
|
||||
explicit identifier(const std::string & val) : val(val), key(mk_val<value_string>(val)) {}
|
||||
explicit identifier(const std::string & val) : val(val) {}
|
||||
std::string type() const override { return "Identifier"; }
|
||||
value execute_impl(context & ctx) const override;
|
||||
};
|
||||
@@ -412,31 +373,28 @@ struct identifier : public expression {
|
||||
|
||||
struct integer_literal : public expression {
|
||||
int64_t val;
|
||||
const value constant; // see cached_val()
|
||||
explicit integer_literal(int64_t val) : val(val), constant(mk_val<value_int>(val)) {}
|
||||
explicit integer_literal(int64_t val) : val(val) {}
|
||||
std::string type() const override { return "IntegerLiteral"; }
|
||||
value execute_impl(context & ctx) const override {
|
||||
return cached_val<value_int>(ctx, constant);
|
||||
value execute_impl(context &) const override {
|
||||
return mk_val<value_int>(val);
|
||||
}
|
||||
};
|
||||
|
||||
struct float_literal : public expression {
|
||||
double val;
|
||||
const value constant; // see cached_val()
|
||||
explicit float_literal(double val) : val(val), constant(mk_val<value_float>(val)) {}
|
||||
explicit float_literal(double val) : val(val) {}
|
||||
std::string type() const override { return "FloatLiteral"; }
|
||||
value execute_impl(context & ctx) const override {
|
||||
return cached_val<value_float>(ctx, constant);
|
||||
value execute_impl(context &) const override {
|
||||
return mk_val<value_float>(val);
|
||||
}
|
||||
};
|
||||
|
||||
struct string_literal : public expression {
|
||||
std::string val;
|
||||
const value constant; // see cached_val()
|
||||
explicit string_literal(const std::string & val) : val(val), constant(mk_val<value_string>(val)) {}
|
||||
explicit string_literal(const std::string & val) : val(val) {}
|
||||
std::string type() const override { return "StringLiteral"; }
|
||||
value execute_impl(context & ctx) const override {
|
||||
return cached_val<value_string>(ctx, constant);
|
||||
value execute_impl(context &) const override {
|
||||
return mk_val<value_string>(val);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -510,28 +468,6 @@ struct binary_expression : public expression {
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Two or more comparisons in a row, e.g. `a < b <= c`
|
||||
* It is the same as `a < b and b <= c`, but each operand is executed only one time.
|
||||
*/
|
||||
struct comparison_chain_expression : public expression {
|
||||
statements operands;
|
||||
std::vector<token> ops;
|
||||
|
||||
comparison_chain_expression(statements && operands, std::vector<token> && ops)
|
||||
: operands(std::move(operands)), ops(std::move(ops)) {
|
||||
assert(this->operands.size() == this->ops.size() + 1);
|
||||
for (const auto & operand : this->operands) chk_type<expression>(operand);
|
||||
}
|
||||
std::string type() const override { return "ComparisonChainExpression"; }
|
||||
value execute_impl(context & ctx) const override;
|
||||
void visit(context & ctx) const override {
|
||||
ctx.visitor(false, this, {
|
||||
{"operands", stmts_to_ptr(operands)}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* An operation with two sides, separated by the | operator.
|
||||
* Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202
|
||||
@@ -694,9 +630,7 @@ struct keyword_argument_expression : public expression {
|
||||
|
||||
struct spread_expression : public expression {
|
||||
statement_ptr argument;
|
||||
bool is_kwargs;
|
||||
spread_expression(statement_ptr && argument, bool is_kwargs)
|
||||
: argument(std::move(argument)), is_kwargs(is_kwargs) {
|
||||
explicit spread_expression(statement_ptr && argument) : argument(std::move(argument)) {
|
||||
chk_type<expression>(this->argument);
|
||||
}
|
||||
std::string type() const override { return "SpreadExpression"; }
|
||||
@@ -776,7 +710,7 @@ struct rethrown_exception : public std::exception {
|
||||
|
||||
//////////////////////
|
||||
|
||||
static void gather_string_parts_recursive(const value & val, value_string & parts, int depth = 0) {
|
||||
static void gather_string_parts_recursive(const value & val, value_string & parts) {
|
||||
// TODO: probably allow print value_none as "None" string? currently this breaks some templates
|
||||
if (is_val<value_string>(val)) {
|
||||
const auto & str_val = cast_val<value_string>(val)->val_str;
|
||||
@@ -784,12 +718,10 @@ static void gather_string_parts_recursive(const value & val, value_string & part
|
||||
} else if (is_val<value_int>(val) || is_val<value_float>(val) || is_val<value_bool>(val)) {
|
||||
std::string str_val = val->as_string().str();
|
||||
parts->val_str.append(str_val);
|
||||
} else if (is_val<value_array>(val) && depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
} else if (is_val<value_array>(val)) {
|
||||
const auto & items = cast_val<value_array>(val)->as_array();
|
||||
auto items = cast_val<value_array>(val)->as_array();
|
||||
for (const auto & item : items) {
|
||||
gather_string_parts_recursive(item, parts, depth + 1);
|
||||
gather_string_parts_recursive(item, parts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+6
-51
@@ -43,59 +43,14 @@ void string::mark_input() {
|
||||
}
|
||||
|
||||
std::string string::str() const {
|
||||
std::string result;
|
||||
result.reserve(length());
|
||||
if (parts.size() == 1) {
|
||||
return parts[0].val;
|
||||
}
|
||||
std::ostringstream oss;
|
||||
for (const auto & part : parts) {
|
||||
result += part.val;
|
||||
oss << part.val;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool string::empty() const {
|
||||
for (const auto & part : parts) {
|
||||
if (!part.val.empty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<char> string::byte_at(size_t index) const {
|
||||
for (const auto & part : parts) {
|
||||
if (index < part.val.size()) {
|
||||
return part.val[index];
|
||||
}
|
||||
index -= part.val.size();
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
bool string::equals(const string & other) const {
|
||||
size_t part = 0, offset = 0;
|
||||
for (const auto & left : parts) {
|
||||
std::string_view text(left.val);
|
||||
while (!text.empty()) {
|
||||
while (part < other.parts.size() && offset == other.parts[part].val.size()) {
|
||||
++part;
|
||||
offset = 0;
|
||||
}
|
||||
if (part == other.parts.size()) {
|
||||
return false;
|
||||
}
|
||||
const auto & right = other.parts[part].val;
|
||||
const auto count = std::min(text.size(), right.size() - offset);
|
||||
if (text.substr(0, count) != std::string_view(right).substr(offset, count)) {
|
||||
return false;
|
||||
}
|
||||
text.remove_prefix(count);
|
||||
offset += count;
|
||||
}
|
||||
}
|
||||
while (part < other.parts.size() && offset == other.parts[part].val.size()) {
|
||||
++part;
|
||||
offset = 0;
|
||||
}
|
||||
return part == other.parts.size();
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
size_t string::length() const {
|
||||
|
||||
@@ -39,9 +39,6 @@ struct string {
|
||||
|
||||
std::string str() const;
|
||||
size_t length() const;
|
||||
bool empty() const;
|
||||
bool equals(const string & other) const;
|
||||
std::optional<char> byte_at(size_t index) const;
|
||||
void hash_update(hasher & hash) const noexcept;
|
||||
bool all_parts_are_input() const;
|
||||
bool is_uppercase() const;
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
#define JINJA_RECURSIVE_LIMIT 128
|
||||
|
||||
namespace jinja {
|
||||
|
||||
static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {
|
||||
|
||||
Loaded 100 of 2806 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user