Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5751c977b7 | ||
|
|
64543a435b | ||
|
|
e70486447b | ||
|
|
3d066892a0 | ||
|
|
85ebb674e0 | ||
|
|
c65e81f30f | ||
|
|
031fc95095 | ||
|
|
7f800dab94 | ||
|
|
16fd90ef37 | ||
|
|
ff8715b989 | ||
|
|
eaa86c462a | ||
|
|
175eb7ac30 | ||
|
|
e7863b1a5a | ||
|
|
4b553876cd | ||
|
|
c68f58f251 | ||
|
|
ff53c0121c | ||
|
|
9d266b5317 | ||
|
|
a087e5d587 | ||
|
|
dffaa6e598 | ||
|
|
aeac8898d5 | ||
|
|
1e487057b8 | ||
|
|
adcf9f2c2a | ||
|
|
bb1746dd72 | ||
|
|
97b91c7bec | ||
|
|
b77fb98acb |
No files matched your search
+6
-11
@@ -19,16 +19,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
@@ -44,7 +40,7 @@ WORKDIR /app
|
||||
# -- Copy project files --
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
# -- Set CANN environment variables (required for compilation) --
|
||||
# Using ENV instead of `source` allows environment variables to persist across the entire image layer
|
||||
@@ -149,7 +145,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
# ==============================================================================
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
ENTRYPOINT [ "/app/llama-cli" ]
|
||||
|
||||
@@ -159,9 +155,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
HEALTHCHECK --interval=5m CMD [ "curl", "-f", "http://localhost:8080/health" ]
|
||||
|
||||
|
||||
+10
-13
@@ -1,4 +1,4 @@
|
||||
ARG UBUNTU_VERSION=26.04
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
ARG APP_REVISION=N/A
|
||||
@@ -9,16 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -26,13 +22,15 @@ FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential git cmake libssl-dev
|
||||
apt-get install -y gcc-14 g++-14 build-essential git cmake libssl-dev
|
||||
|
||||
ENV CC=gcc-14 CXX=g++-14
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
|
||||
@@ -106,7 +104,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
### Light, CLI only
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -116,9 +114,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
+6
-14
@@ -17,16 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
@@ -44,7 +40,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
|
||||
@@ -88,9 +84,6 @@ RUN apt-get update \
|
||||
&& find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
|
||||
&& find /var/cache -type f -delete
|
||||
|
||||
# Some hosts (e.g. GKE) only mount the driver here, and CUDA 12.8.1-12.9.1 base images don't set these paths
|
||||
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/nvidia/lib:${LD_LIBRARY_PATH}
|
||||
|
||||
COPY --from=build /app/lib/ /app
|
||||
|
||||
### Full
|
||||
@@ -120,7 +113,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
### Light, CLI only
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -130,9 +123,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
+21
-33
@@ -1,4 +1,4 @@
|
||||
ARG ONEAPI_VERSION=2026.2.0-devel-ubuntu24.04
|
||||
ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
ARG APP_REVISION=N/A
|
||||
@@ -11,41 +11,32 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
|
||||
RUN echo "Download oneDNN installation package" && \
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh && \
|
||||
echo "Install oneDNN components" && \
|
||||
bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS build
|
||||
|
||||
ARG GGML_SYCL_F16=ON
|
||||
ARG LEVEL_ZERO_VERSION=1.34.0
|
||||
ARG LEVEL_ZERO_VERSION=1.28.2
|
||||
ARG LEVEL_ZERO_UBUNTU_VERSION=u24.04
|
||||
RUN apt-get update && \
|
||||
apt-get install -y git libssl-dev wget ca-certificates && \
|
||||
cd /tmp && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb && \
|
||||
apt-get -o Dpkg::Options::="--force-overwrite" install -y ./libze1.deb ./libze-dev.deb && \
|
||||
rm -f /tmp/libze1.deb /tmp/libze-dev.deb
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb && \
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb && \
|
||||
apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb && \
|
||||
rm -f /tmp/level-zero.deb /tmp/level-zero-devel.deb
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
echo "GGML_SYCL_F16 is set" \
|
||||
@@ -54,7 +45,7 @@ RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
fi && \
|
||||
echo "Building with dynamic libs" && \
|
||||
cmake -B build -DGGML_NATIVE=OFF -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DLLAMA_BUILD_TESTS=OFF ${OPT_SYCL_F16} && \
|
||||
cmake --build build --config Release -j$((($(nproc)+1)/2))
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
|
||||
RUN mkdir -p /app/lib && \
|
||||
find build -name "*.so*" -exec cp -P {} /app/lib \;
|
||||
@@ -68,7 +59,7 @@ RUN mkdir -p /app/full \
|
||||
&& cp requirements.txt /app/full \
|
||||
&& cp .devops/tools.sh /app/full/tools.sh
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
|
||||
FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
|
||||
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
@@ -83,8 +74,6 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
|
||||
org.opencontainers.image.url=$IMAGE_URL \
|
||||
org.opencontainers.image.source=$IMAGE_SOURCE
|
||||
|
||||
COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
|
||||
|
||||
#Following versions are for multiple GPUs, since 26.x has known issue:
|
||||
# https://github.com/ggml-org/llama.cpp/issues/21747,
|
||||
# https://github.com/intel/compute-runtime/issues/921.
|
||||
@@ -95,11 +84,11 @@ COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
|
||||
#ARG IGDGMM_VERSION=22.8.2
|
||||
|
||||
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
ARG IGC_VERSION=v2.34.4
|
||||
ARG IGC_VERSION_FULL=2_2.34.4+21428
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
|
||||
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
|
||||
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
|
||||
@@ -113,7 +102,7 @@ RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
|
||||
&& dpkg --install *.deb
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y libgomp1 curl ffmpeg openssl libssl-dev \
|
||||
&& apt-get install -y libgomp1 curl ffmpeg \
|
||||
&& apt autoremove -y \
|
||||
&& apt clean -y \
|
||||
&& rm -rf /tmp/* /var/tmp/* \
|
||||
@@ -152,7 +141,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/lib/ /app
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -162,10 +151,9 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/lib/ /app
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
+6
-11
@@ -15,16 +15,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
@@ -49,7 +45,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
|
||||
@@ -124,7 +120,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
### Light, CLI only
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -134,9 +130,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
+41
-138
@@ -22,19 +22,12 @@
|
||||
spirv-headers,
|
||||
nodejs,
|
||||
importNpmLock,
|
||||
syclToolkit ? null,
|
||||
level-zero,
|
||||
ocl-icd,
|
||||
intel-graphics-compiler,
|
||||
intel-compute-runtime,
|
||||
makeBinaryWrapper,
|
||||
useBlas ?
|
||||
builtins.all (x: !x) [
|
||||
useCuda
|
||||
useMetalKit
|
||||
useRocm
|
||||
useVulkan
|
||||
useSycl
|
||||
]
|
||||
&& blas.meta.available,
|
||||
useCuda ? config.cudaSupport,
|
||||
@@ -45,28 +38,11 @@
|
||||
rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,
|
||||
useVulkan ? false,
|
||||
useRpc ? false,
|
||||
useSycl ? false,
|
||||
# Rebuild, not a runtime switch. cmake default is off.
|
||||
syclF16 ? false,
|
||||
# null keeps JIT. AOT makes icpx run ocloc, which DPC++ does not ship.
|
||||
syclDeviceArch ? null,
|
||||
syclMaxParallelLinkJobs ? null,
|
||||
llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
|
||||
|
||||
# CUDA needs its backend stdenv or libstdc++ mismatches downstream.
|
||||
# SYCL needs icpx which is provided by the oneAPI stdenv
|
||||
effectiveStdenv ?
|
||||
if useCuda && useSycl then
|
||||
throw "llama-cpp: useCuda and useSycl are mutually exclusive"
|
||||
else if useCuda then
|
||||
cudaPackages.backendStdenv
|
||||
else if useSycl then
|
||||
if syclToolkit == null then
|
||||
throw "llama-cpp: useSycl requires syclToolkit, consider setting it to intel-oneapi-toolkit"
|
||||
else
|
||||
syclToolkit.stdenv
|
||||
else
|
||||
stdenv,
|
||||
# It's necessary to consistently use backendStdenv when building with CUDA support,
|
||||
# otherwise we get libstdc++ errors downstream.
|
||||
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
|
||||
enableStatic ? effectiveStdenv.hostPlatform.isStatic,
|
||||
precompileMetalShaders ? false,
|
||||
useWebUi ? true,
|
||||
@@ -89,8 +65,7 @@ let
|
||||
++ lib.optionals useMetalKit [ "MetalKit" ]
|
||||
++ lib.optionals useMpi [ "MPI" ]
|
||||
++ lib.optionals useRocm [ "ROCm" ]
|
||||
++ lib.optionals useVulkan [ "Vulkan" ]
|
||||
++ lib.optionals useSycl [ "SYCL" ];
|
||||
++ lib.optionals useVulkan [ "Vulkan" ];
|
||||
|
||||
pnameSuffix =
|
||||
strings.optionalString (suffices != [ ])
|
||||
@@ -133,26 +108,6 @@ let
|
||||
shaderc
|
||||
spirv-headers
|
||||
];
|
||||
|
||||
syclAot = useSycl && syclDeviceArch != null && syclDeviceArch != "";
|
||||
|
||||
# source ocloc binary from intel-compute-runtime
|
||||
oclocAot = runCommand "ocloc-aot" { nativeBuildInputs = [ makeBinaryWrapper ]; } ''
|
||||
mkdir -p "$out/bin"
|
||||
shopt -s nullglob
|
||||
exact=(${lib.getBin intel-compute-runtime}/bin/ocloc)
|
||||
vers=(${lib.getBin intel-compute-runtime}/bin/ocloc-*)
|
||||
if [ ''${#exact[@]} -eq 1 ]; then
|
||||
src=''${exact[0]}
|
||||
elif [ ''${#vers[@]} -eq 1 ]; then
|
||||
src=''${vers[0]}
|
||||
else
|
||||
echo "expected one ocloc binary from intel-compute-runtime" >&2
|
||||
exit 1
|
||||
fi
|
||||
makeWrapper "$src" "$out/bin/ocloc" \
|
||||
--prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ intel-graphics-compiler ]}
|
||||
'';
|
||||
in
|
||||
|
||||
effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
@@ -178,13 +133,11 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
src = lib.cleanSource ../../.;
|
||||
};
|
||||
|
||||
# Builds Llama Chat locally, taking care not to require updating any sha256 hash.
|
||||
# Builds the webui locally, taking care not to require updating any sha256 hash.
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
# the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole
|
||||
src = lib.cleanSource ../../ui;
|
||||
sourceRoot = "source/apps/chat";
|
||||
src = lib.cleanSource ../../tools/ui;
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -193,7 +146,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# no sha256 required when using buildNodeModules
|
||||
npmDeps = importNpmLock.buildNodeModules {
|
||||
npmRoot = ../../ui/apps/chat;
|
||||
npmRoot = ../../tools/ui;
|
||||
inherit nodejs;
|
||||
};
|
||||
|
||||
@@ -203,8 +156,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
};
|
||||
|
||||
postPatch = lib.optionalString useWebUi ''
|
||||
cp -r ${finalAttrs.webui} ui/apps/chat/dist
|
||||
chmod -R u+w ui/apps/chat/dist
|
||||
cp -r ${finalAttrs.webui} tools/ui/dist
|
||||
chmod -R u+w tools/ui/dist
|
||||
'';
|
||||
|
||||
# With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
|
||||
@@ -215,13 +168,6 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
# see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
|
||||
__noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
|
||||
# Do not fortify under SYCL as it breaks compiled memcpy invocations
|
||||
# _FORTIFY_SOURCE rewrites memcpy -> __memcpy_chk
|
||||
hardeningDisable = optionals useSycl [
|
||||
"fortify"
|
||||
"fortify3"
|
||||
];
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
cmake
|
||||
@@ -235,12 +181,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
autoAddDriverRunpath
|
||||
]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]
|
||||
++ optionals useSycl [
|
||||
autoAddDriverRunpath
|
||||
makeBinaryWrapper
|
||||
]
|
||||
++ optionals syclAot [ oclocAot ];
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
|
||||
buildInputs =
|
||||
optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
|
||||
@@ -249,53 +190,39 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
++ optionals useRocm rocmBuildInputs
|
||||
++ optionals useBlas [ blas ]
|
||||
++ optionals useVulkan vulkanBuildInputs
|
||||
++ optionals useSycl [
|
||||
level-zero
|
||||
ocl-icd
|
||||
intel-graphics-compiler
|
||||
]
|
||||
++ [ openssl ];
|
||||
|
||||
cmakeFlags = [
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
cmakeFlags =
|
||||
[
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
)
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
]
|
||||
++ optionals useSycl [
|
||||
(cmakeBool "GGML_SYCL" true)
|
||||
(cmakeBool "GGML_SYCL_F16" syclF16)
|
||||
]
|
||||
++ optionals syclAot [
|
||||
(cmakeFeature "GGML_SYCL_DEVICE_ARCH" syclDeviceArch)
|
||||
]
|
||||
++ optionals (syclAot && syclMaxParallelLinkJobs != null) [
|
||||
(cmakeFeature "GGML_SYCL_MAX_PARALLEL_LINK_JOBS" (toString syclMaxParallelLinkJobs))
|
||||
];
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
];
|
||||
|
||||
# Environment variables needed for ROCm
|
||||
env = optionalAttrs useRocm {
|
||||
@@ -310,35 +237,11 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
cp $src/include/llama.h $out/include/
|
||||
'';
|
||||
|
||||
# Fix SYCL binary calling missing important libraries
|
||||
# Also use Level Zero + sysman INSTEAD of OpenCL
|
||||
# due to OpenCL reporting free memory as total memory,
|
||||
# breaking `-fit` functionality
|
||||
postFixup = lib.optionalString useSycl ''
|
||||
driverLibs=${
|
||||
lib.makeLibraryPath [
|
||||
intel-graphics-compiler
|
||||
ocl-icd
|
||||
]
|
||||
}
|
||||
for b in "$out"/bin/*; do
|
||||
[ -f "$b" ] && [ -x "$b" ] || continue
|
||||
patchelf --print-interpreter "$b" >/dev/null 2>&1 || continue
|
||||
patchelf --add-needed libOpenCL.so.1 --add-needed libigc.so.2 \
|
||||
--add-rpath "$driverLibs" "$b"
|
||||
wrapProgram "$b" \
|
||||
--set-default ONEAPI_DEVICE_SELECTOR level_zero:gpu \
|
||||
--set-default ZES_ENABLE_SYSMAN 1
|
||||
done
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# Configurations we don't want even the CI to evaluate. Results in the
|
||||
# "unsupported platform" messages. This is mostly a no-op, because
|
||||
# cudaPackages would've refused to evaluate anyway.
|
||||
badPlatforms =
|
||||
optionals useCuda lib.platforms.darwin
|
||||
++ optionals useSycl (lib.subtractLists [ "x86_64-linux" ] lib.platforms.all);
|
||||
badPlatforms = optionals useCuda lib.platforms.darwin;
|
||||
|
||||
# Configurations that are known to result in build failures. Can be
|
||||
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||
|
||||
@@ -22,6 +22,10 @@ let
|
||||
gitpython
|
||||
tabulate
|
||||
|
||||
# for examples/pydantic-models-to-grammar-examples.py
|
||||
docstring-parser
|
||||
pydantic
|
||||
|
||||
];
|
||||
|
||||
llama-python-test-deps = with python3Packages; [
|
||||
|
||||
@@ -3,11 +3,11 @@ ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
ARG IGC_VERSION=v2.41.5
|
||||
ARG IGC_VERSION_FULL=2_2.41.5+22716
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
ARG NPU_DRIVER_VERSION=v1.38.0
|
||||
@@ -28,16 +28,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
@@ -87,7 +83,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
# Build Stage
|
||||
RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
|
||||
@@ -231,7 +227,6 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
|
||||
|
||||
+6
-11
@@ -17,16 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
@@ -56,7 +52,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
||||
cmake -S . -B build \
|
||||
@@ -130,7 +126,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
### Light, CLI only
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -140,9 +136,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
@@ -133,7 +133,6 @@ ENTRYPOINT [ "/llama.cpp/bin/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
WORKDIR /llama.cpp/bin
|
||||
|
||||
|
||||
+17
-16
@@ -9,16 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -35,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
@@ -86,14 +82,20 @@ COPY --from=build /app/full /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}"
|
||||
|
||||
# Flag for compatibility with pip
|
||||
ARG UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y \
|
||||
build-essential \
|
||||
curl \
|
||||
git \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
&& pip install --break-system-packages --upgrade setuptools \
|
||||
&& pip install --break-system-packages -r requirements.txt \
|
||||
ca-certificates \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
|
||||
&& uv python install 3.13 \
|
||||
&& uv venv --python 3.13 /root/.venv \
|
||||
&& uv pip install --python /root/.venv/bin/python -r requirements.txt \
|
||||
&& apt autoremove -y \
|
||||
&& apt clean -y \
|
||||
&& rm -rf /tmp/* /var/tmp/* \
|
||||
@@ -105,7 +107,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
### Light, CLI only
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -115,9 +117,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
@@ -9,16 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/ui/apps/chat
|
||||
WORKDIR /app/tools/ui
|
||||
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI and Llama Voice from their sources next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
COPY ui/lib/voice/src/ /app/ui/lib/voice/src/
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -32,7 +28,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \
|
||||
cmake --build build -j $(nproc)
|
||||
@@ -101,7 +97,7 @@ ENTRYPOINT ["/app/tools.sh"]
|
||||
### Light, CLI only
|
||||
FROM base AS light
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -111,9 +107,8 @@ ENTRYPOINT [ "/app/llama-cli" ]
|
||||
FROM base AS server
|
||||
|
||||
ENV LLAMA_ARG_HOST=0.0.0.0
|
||||
ENV LLAMA_ARG_PORT=8080
|
||||
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app/
|
||||
COPY --from=build /app/full/llama /app/full/llama-server /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
build*/
|
||||
|
||||
ui/apps/chat/node_modules/
|
||||
tools/ui/node_modules/
|
||||
|
||||
models/*
|
||||
|
||||
|
||||
+1
-4
@@ -45,10 +45,7 @@ insert_final_newline = unset
|
||||
trim_trailing_whitespace = unset
|
||||
insert_final_newline = unset
|
||||
|
||||
[vendor/**.patch]
|
||||
trim_trailing_whitespace = unset
|
||||
|
||||
[ui/**]
|
||||
[tools/ui/**]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
end_of_line = unset
|
||||
|
||||
@@ -5,8 +5,6 @@ exclude =
|
||||
# Do not traverse examples and tools
|
||||
examples,
|
||||
tools,
|
||||
# Do not traverse .venv directories
|
||||
.venv*,
|
||||
# Do not include package initializers
|
||||
__init__.py,
|
||||
# No need to traverse our git directory
|
||||
|
||||
@@ -14,9 +14,7 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" && "${{ env.BRANCH_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "name=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
||||
elif [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
@@ -25,9 +25,7 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
@@ -56,9 +54,7 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
@@ -89,9 +85,7 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
@@ -106,60 +100,56 @@ runs:
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.5 for x64
|
||||
if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'x64' }}
|
||||
- name: Install Cuda Toolkit 13.4 for x64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvrtc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-x86_64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvtx-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_profiler_api-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\visual_studio_integration-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.5 for ARM64
|
||||
if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'arm64' }}
|
||||
- name: Install Cuda Toolkit 13.4 for ARM64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-arm64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
+2
-1
@@ -48,6 +48,7 @@ examples:
|
||||
- all-globs-to-all-files:
|
||||
- '!tools/server/**'
|
||||
- '!tools/mtmd/**'
|
||||
- '!tools/ui/**'
|
||||
testing:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -71,7 +72,7 @@ android:
|
||||
server/ui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- ui/**
|
||||
- tools/ui/**
|
||||
server:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
name: Behavior Check
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/behavior-check.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.py',
|
||||
'**/requirements*.txt',
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/behavior-check.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.py',
|
||||
'**/requirements*.txt',
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
GGML_SCHED_DEBUG_REALLOC: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential libssl-dev git-lfs libjpeg-dev python3 python3-venv python3-pip
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: behavior-check-ubuntu-26.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_BACKEND_DL=OFF \
|
||||
-DLLAMA_FATAL_WARNINGS=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: behavior-check-ubuntu-26.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
cd build
|
||||
ctest -L behavior -LE 'model|backend' --verbose --timeout 900
|
||||
|
||||
windows:
|
||||
runs-on: windows-2025
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
key: behavior-check-windows-2025
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
shell: bash
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
run: |
|
||||
call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
|
||||
cmake -B build -G "Ninja Multi-Config" ^
|
||||
-DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
|
||||
-DCMAKE_BUILD_TYPE=Release ^
|
||||
-DGGML_NATIVE=OFF ^
|
||||
-DGGML_BACKEND_DL=OFF
|
||||
set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
|
||||
cmake --build build --config Release -j %NINJA_JOBS%
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
shell: bash
|
||||
run: |
|
||||
source .venv/Scripts/activate
|
||||
cd build
|
||||
ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: behavior-check-windows-2025
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
@@ -31,8 +31,8 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-llguidance:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
ubuntu-24-llguidance:
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -33,9 +33,9 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
android-ndk-snapdragon:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8'
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -67,9 +67,9 @@ jobs:
|
||||
path: pkg-snapdragon/llama.cpp
|
||||
|
||||
linux-iot-snapdragon:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8'
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -103,12 +103,12 @@ jobs:
|
||||
test-snapdragon-qdc:
|
||||
name: Test on QDC Device (${{ matrix.device }})
|
||||
needs: [android-ndk-snapdragon, linux-iot-snapdragon]
|
||||
runs-on: ubuntu-26.04-arm
|
||||
timeout-minutes: 180
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
device: [SM8850, QCS9075M]
|
||||
device: [SM8750, SM8850, QCS9075M]
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
@@ -155,7 +155,7 @@ jobs:
|
||||
--pkg-dir pkg-snapdragon/llama.cpp \
|
||||
--model-url "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \
|
||||
--device ${{ matrix.device }} \
|
||||
${{ startsWith(matrix.device, 'QCS') && '--job-timeout 5400 --retries 2 --retry-delay 300' || '' }}
|
||||
${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }}
|
||||
env:
|
||||
QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ env:
|
||||
|
||||
jobs:
|
||||
default:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -56,7 +56,6 @@ jobs:
|
||||
uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
|
||||
with:
|
||||
log-accepted-android-sdk-licenses: false
|
||||
packages: platform-tools cmake;3.31.6
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
@@ -64,7 +63,7 @@ jobs:
|
||||
./gradlew build --no-daemon
|
||||
|
||||
ndk:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
|
||||
defaults:
|
||||
@@ -102,7 +101,7 @@ jobs:
|
||||
path: pkg-adb/llama.cpp
|
||||
|
||||
arm64:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
env:
|
||||
NDK_VERSION: "29.0.14206865"
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
name: CI (apple - daily)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Xcode
|
||||
uses: ggml-org/setup-xcode@v1
|
||||
with:
|
||||
xcode-version: latest-stable
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
|
||||
- name: Upload xcframework artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
retention-days: 1
|
||||
|
||||
- name: Build Xcode project
|
||||
run: |
|
||||
xcodebuild -downloadPlatform iOS
|
||||
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
|
||||
macos-latest-tvos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-visionos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-swift:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download xcframework artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
|
||||
- name: Build llama.cpp with CMake
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
@@ -105,5 +105,198 @@ jobs:
|
||||
cd build
|
||||
# Metal Paravirtual devices are difficult to support -> disable
|
||||
# ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
|
||||
ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
|
||||
ctest -L main -E "test-llama-archs|test-save-load-state|test-recurrent-state-rollback" --verbose --timeout 900
|
||||
|
||||
macos-latest-x64:
|
||||
runs-on: macos-15-intel
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: apple-x64
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
# Metal is disabled due to intermittent failures with Github runners not having a GPU:
|
||||
# https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_RPATH="@loader_path" \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_BUILD_BORINGSSL=ON \
|
||||
-DGGML_METAL=OFF \
|
||||
-DGGML_RPC=ON \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
|
||||
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: apple-x64
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Xcode
|
||||
uses: ggml-org/setup-xcode@v1
|
||||
with:
|
||||
xcode-version: latest-stable
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
|
||||
- name: Upload xcframework artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
retention-days: 1
|
||||
|
||||
- name: Build Xcode project
|
||||
run: |
|
||||
xcodebuild -downloadPlatform iOS
|
||||
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
|
||||
macos-latest-tvos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-visionos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-swift:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download xcframework artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
|
||||
- name: Build llama.cpp with CMake
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
@@ -40,11 +40,11 @@ jobs:
|
||||
# path: ./spacemit_toolchain
|
||||
# version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
|
||||
|
||||
ubuntu-26-openvino-cache:
|
||||
runs-on: ubuntu-26.04
|
||||
ubuntu-24-openvino-cache:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
runs-on: windows-2022
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Build relocatable cmake package
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
runs-on: [self-hosted, Linux, CPU]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
PREFIX="$(pwd)"/inst
|
||||
cmake -S . -B build \
|
||||
-DCMAKE_PREFIX_PATH="$PREFIX" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_IS_DEV=OFF \
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
cmake --install build --prefix "$PREFIX" --config Release
|
||||
|
||||
export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
|
||||
build_commit=$(git rev-parse --short HEAD | xargs)
|
||||
build_number=$(git rev-list --count HEAD | xargs)
|
||||
|
||||
major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
build_version="$major.$minor.$patch"
|
||||
|
||||
checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
|
||||
"set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
|
||||
"set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
|
||||
|
||||
for check in "${checks[@]}"; do
|
||||
if ! grep -qE "$check" "$LLAMA_CONFIG"; then
|
||||
echo "Checking llama-config.cmake version... \"$check\" failed!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Checking llama-config.cmake version... success."
|
||||
|
||||
cd examples/simple-cmake-pkg
|
||||
cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
|
||||
cmake --build build -j $(nproc)
|
||||
@@ -7,6 +7,7 @@ on:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-cpu.yml',
|
||||
'.github/workflows/build-cmake-pkg.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
@@ -19,6 +20,7 @@ on:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-cpu.yml',
|
||||
'.github/workflows/build-cmake-pkg.yml',
|
||||
'ggml/src/ggml-rpc/**',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
@@ -45,14 +47,17 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
build-cmake-pkg:
|
||||
uses: ./.github/workflows/build-cmake-pkg.yml
|
||||
|
||||
ubuntu:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- build: 'x64'
|
||||
os: ubuntu-26.04
|
||||
os: ubuntu-22.04
|
||||
- build: 'arm64'
|
||||
os: ubuntu-26.04-arm
|
||||
os: ubuntu-24.04-arm
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@@ -76,6 +81,13 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
@@ -117,7 +129,7 @@ jobs:
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
ctest -L 'main|python' --verbose --timeout 900
|
||||
|
||||
- name: Test llama2c conversion
|
||||
id: llama2c_test
|
||||
@@ -187,11 +199,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
@@ -235,79 +242,3 @@ jobs:
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
build-cmake-pkg:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: build-cmake-pkg
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
PREFIX="$(pwd)"/inst
|
||||
cmake -S . -B build \
|
||||
-DCMAKE_PREFIX_PATH="$PREFIX" \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_IS_DEV=OFF \
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
cmake --install build --prefix "$PREFIX" --config Release
|
||||
|
||||
export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
|
||||
build_commit=$(git rev-parse --short HEAD | xargs)
|
||||
build_number=$(git rev-list --count HEAD | xargs)
|
||||
|
||||
major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
||||
build_version="$major.$minor.$patch"
|
||||
|
||||
checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
|
||||
"set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
|
||||
"set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
|
||||
|
||||
for check in "${checks[@]}"; do
|
||||
if ! grep -qE "$check" "$LLAMA_CONFIG"; then
|
||||
echo "Checking llama-config.cmake version... \"$check\" failed!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Checking llama-config.cmake version... success."
|
||||
|
||||
cd examples/simple-cmake-pkg
|
||||
cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: build-cmake-pkg
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
@@ -163,7 +163,7 @@ jobs:
|
||||
# cmake --build build --config Release -j $(nproc)
|
||||
|
||||
debian-13-loongarch64-cpu-cross:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
|
||||
|
||||
steps:
|
||||
@@ -218,7 +218,7 @@ jobs:
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
debian-13-loongarch64-vulkan-cross:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
|
||||
|
||||
steps:
|
||||
@@ -276,8 +276,8 @@ jobs:
|
||||
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
ubuntu-26-riscv64-cpu-spacemit-ime-cross:
|
||||
runs-on: ubuntu-26.04
|
||||
ubuntu-24-riscv64-cpu-spacemit-ime-cross:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
# Make sure this is in sync with build-cache.yml
|
||||
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
|
||||
hip:
|
||||
runs-on: ubuntu-22.04
|
||||
container: rocm/dev-ubuntu-22.04:6.3.4
|
||||
container: rocm/dev-ubuntu-22.04:6.1.2
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -40,12 +40,12 @@ jobs:
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.5'
|
||||
- cuda: '13.4'
|
||||
arch: x64
|
||||
defines: ''
|
||||
- cuda: '13.5'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.4'
|
||||
arch: arm64
|
||||
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -20,11 +20,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/build-ibm.yml',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-zdnn/**',
|
||||
'ggml/src/ggml-quants.c',
|
||||
'src/llama-quant.cpp',
|
||||
'gguf-py/gguf/scripts/gguf_convert_endian.py',
|
||||
'tests/test-quantize-be.sh'
|
||||
'ggml/src/ggml-zdnn/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
@@ -43,9 +39,9 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-s390x:
|
||||
name: ubuntu-26-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
ubuntu-24-s390x:
|
||||
name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -66,6 +62,12 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
@@ -100,29 +102,35 @@ jobs:
|
||||
run: |
|
||||
cd build
|
||||
echo "Fetch llama2c big-endian model"
|
||||
wget https://huggingface.co/ggml-org-dev/stories260K/resolve/main/stories260K-be.gguf
|
||||
wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
|
||||
./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
||||
|
||||
- name: Test quantize (s390x)
|
||||
id: quantize_test_s390x
|
||||
run: |
|
||||
./tests/test-quantize-be.sh ./build/bin/llama-quantize ./build/bin/llama-perplexity 2>&1
|
||||
|
||||
ubuntu-26-zdnn-s390x:
|
||||
name: ubuntu-26-zdnn-s390x
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
container: ubuntu:26.04 # required to get GCC 15.1 and binutils 2.44
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates \
|
||||
libssl-dev libzdnn-dev
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
- name: Toolchain workaround (GCC 15)
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential libssl-dev libzdnn-dev
|
||||
apt-get install -y gcc-15 g++-15
|
||||
echo "CC=gcc-15" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-15" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build with zDNN Backend
|
||||
id: cmake_build
|
||||
@@ -137,8 +145,8 @@ jobs:
|
||||
-DCMAKE_CXX_FLAGS="-march=arch15"
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
ubuntu-26-ppc64le:
|
||||
runs-on: ubuntu-26.04-ppc64le
|
||||
ubuntu-24-ppc64le:
|
||||
runs-on: ubuntu-24.04-ppc64le
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -154,6 +162,12 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
|
||||
@@ -61,11 +61,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -37,9 +37,64 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXCLUDE: "test-llama-archs|test-llama-context|test-fp8-kv"
|
||||
CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
|
||||
|
||||
jobs:
|
||||
ubuntu-24-openvino:
|
||||
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip
|
||||
sudo apt-get install -y ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
|
||||
- name: Setup OpenVINO Toolkit
|
||||
uses: ./.github/actions/linux-setup-openvino
|
||||
with:
|
||||
path: ./openvino_toolkit
|
||||
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
||||
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
||||
|
||||
- name: Install OpenVINO dependencies
|
||||
run: |
|
||||
cd ./openvino_toolkit
|
||||
chmod +x ./install_dependencies/install_openvino_dependencies.sh
|
||||
echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
source ./openvino_toolkit/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_OPENVINO=ON
|
||||
time cmake --build build/ReleaseOV --config Release --parallel
|
||||
|
||||
- name: Test (CPU)
|
||||
id: cmake_test_cpu
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
|
||||
- name: Test (GPU)
|
||||
id: cmake_test_gpu
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
|
||||
openvino-windows-2022:
|
||||
runs-on: windows-2022
|
||||
|
||||
@@ -49,7 +104,7 @@ jobs:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
|
||||
@@ -43,12 +43,13 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# thread and address doesn't run properly on some self hosted machines, so run it on Github instead
|
||||
- sanitizer: ADDRESS
|
||||
machine: ubuntu-26.04
|
||||
machine: ubuntu-24.04
|
||||
- sanitizer: THREAD
|
||||
machine: ubuntu-26.04
|
||||
machine: ubuntu-24.04
|
||||
- sanitizer: UNDEFINED
|
||||
machine: ubuntu-26.04
|
||||
machine: [self-hosted, X64, Linux]
|
||||
|
||||
runs-on: ${{ matrix.machine }}
|
||||
|
||||
@@ -57,24 +58,31 @@ jobs:
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# if: ${{ matrix.sanitizer != 'UNDEFINED' }}
|
||||
# with:
|
||||
# key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
|
||||
# variant: ccache
|
||||
# evict-old-files: 1d
|
||||
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: ctest-sanitizer-${{ matrix.sanitizer }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
# with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
|
||||
- name: Build (undefined)
|
||||
id: cmake_build_undefined
|
||||
if: ${{ matrix.sanitizer == 'UNDEFINED' }}
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Debug \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
||||
-DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
|
||||
|
||||
cmake --build build --config Debug -j $(nproc)
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
if: ${{ matrix.sanitizer == 'ADDRESS' || matrix.sanitizer == 'UNDEFINED' }}
|
||||
if: ${{ matrix.sanitizer == 'ADDRESS' }}
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
|
||||
@@ -95,18 +103,6 @@ jobs:
|
||||
|
||||
cmake --build build --config RelWithDebInfo -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: ctest-sanitizer-${{ matrix.sanitizer }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
|
||||
@@ -38,7 +38,7 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-sycl:
|
||||
ubuntu-24-sycl:
|
||||
strategy:
|
||||
matrix:
|
||||
build: [fp32, fp16]
|
||||
@@ -48,12 +48,12 @@ jobs:
|
||||
- build: fp16
|
||||
fp16: ON
|
||||
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
LEVEL_ZERO_VERSION: "1.34.0"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
continue-on-error: true
|
||||
@@ -67,15 +67,8 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cd /tmp
|
||||
echo "Download oneAPI Deep Learning Essentials installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
echo "Install oneAPI Deep Learning Essentials components"
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
echo "Download oneDNN installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh
|
||||
echo "Install oneDNN components"
|
||||
sudo bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
|
||||
sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
|
||||
|
||||
- name: Install Level Zero SDK
|
||||
shell: bash
|
||||
@@ -86,12 +79,6 @@ jobs:
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
|
||||
sudo apt-get install -y ./libze1.deb ./libze-dev.deb
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libssl-dev
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -103,7 +90,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: sycl-ubuntu-26-${{ matrix.build }}
|
||||
key: sycl-ubuntu-24-${{ matrix.build }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -117,6 +104,7 @@ jobs:
|
||||
-DGGML_SYCL=ON \
|
||||
-DCMAKE_C_COMPILER=icx \
|
||||
-DCMAKE_CXX_COMPILER=icpx \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_SYCL_F16=${{ matrix.fp16 }} \
|
||||
-DGGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON \
|
||||
@@ -131,7 +119,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: sycl-ubuntu-26-${{ matrix.build }}
|
||||
key: sycl-ubuntu-24-${{ matrix.build }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
@@ -150,13 +138,11 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
|
||||
WINDOWS_DNNL: intel.oneapi.win.dnnl
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip
|
||||
WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
@@ -165,8 +151,7 @@ jobs:
|
||||
- name: Download & Install oneAPI
|
||||
shell: bash
|
||||
run: |
|
||||
scripts/install-oneapi.bat $WINDOWS_DLE_URL $WINDOWS_DPCPP_MKL
|
||||
scripts/install-oneapi.bat $WINDOWS_ONEDNN_URL $WINDOWS_DNNL
|
||||
scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
|
||||
|
||||
- name: Install Level Zero SDK
|
||||
shell: pwsh
|
||||
|
||||
@@ -31,8 +31,8 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
ubuntu-26-virtgpu:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
ubuntu-24-virtgpu:
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -42,7 +42,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ubuntu-arm64:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -53,7 +53,9 @@ jobs:
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
|
||||
sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -67,7 +69,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-arm
|
||||
key: vulkan-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -90,14 +92,14 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-arm
|
||||
key: vulkan-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
ubuntu-llvmpipe:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -135,7 +137,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-llvmpipe
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -154,7 +156,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: vulkan-ubuntu-26.04-llvmpipe
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
@@ -213,11 +215,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
name: CI (wasi)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-wasi.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.wgsl',
|
||||
'**/*.tmpl',
|
||||
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-wasi.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.wgsl',
|
||||
'**/*.tmpl',
|
||||
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
WASI_SDK_VERSION: 34
|
||||
DAWN_VERSION: v20260908.214631
|
||||
DAWN_COMMIT: 94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655
|
||||
WASI_WEBGPU_HEADERS_COMMIT: 917765edd7cb8f8cf95bda1217514c78f71b582b
|
||||
|
||||
jobs:
|
||||
ubuntu-webgpu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: Install wasi-sdk
|
||||
uses: bytecodealliance/setup-wasi-sdk-action@77d09f3df1e8da344d8403cf8631af8977938b4c # v1.0.0
|
||||
with:
|
||||
version: ${{ env.WASI_SDK_VERSION }}
|
||||
add-to-path: false
|
||||
|
||||
- name: wasi:webgpu Dependency
|
||||
run: |
|
||||
mkdir -p wasi-webgpu/dawn-headers
|
||||
curl -fL "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/dawn-headers-${DAWN_COMMIT}.tar.gz" |
|
||||
tar -xz -C wasi-webgpu/dawn-headers --strip-components=1
|
||||
|
||||
git init -q wasi-webgpu/wasi-webgpu-headers
|
||||
cd wasi-webgpu/wasi-webgpu-headers
|
||||
git fetch -q --depth 1 https://github.com/wasi-gfx/wasi-webgpu-headers.git ${WASI_WEBGPU_HEADERS_COMMIT}
|
||||
git checkout -q FETCH_HEAD
|
||||
git submodule update -q --init --depth 1
|
||||
cd ../..
|
||||
|
||||
# ggml-webgpu does find_package(Dawn) and links dawn::webgpu_dawn. There is no Dawn on WASI,
|
||||
# so this target instead compiles wasi-webgpu-headers (webgpu.h over wasi:webgpu imports)
|
||||
# and exposes Dawn's webgpu_cpp.h on top of it.
|
||||
cat > wasi-webgpu/DawnConfig.cmake << 'EOF'
|
||||
set(_wasi_webgpu "${CMAKE_CURRENT_LIST_DIR}/wasi-webgpu-headers")
|
||||
add_library(dawn::webgpu_dawn INTERFACE IMPORTED)
|
||||
set_target_properties(dawn::webgpu_dawn PROPERTIES
|
||||
INTERFACE_SOURCES "${_wasi_webgpu}/webgpu.c;${_wasi_webgpu}/async_futures.c;${_wasi_webgpu}/imports.c;${_wasi_webgpu}/imports_component_type.o"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/dawn-headers/include/webgpu_upstream;${_wasi_webgpu}")
|
||||
EOF
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasi
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
# llama-common does not build for WASI, so the executables are linked by hand
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
WASI_FLAGS="-msimd128 -D_WASI_EMULATED_SIGNAL -fwasm-exceptions -mllvm -wasm-use-legacy-eh=false"
|
||||
cmake -B build-wasi \
|
||||
-G "Ninja" \
|
||||
-DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_PATH}/share/cmake/wasi-sdk-p3.cmake \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_FLAGS="${WASI_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${WASI_FLAGS}" \
|
||||
-DCMAKE_HAVE_LIBC_PTHREAD=1 \
|
||||
-DBUILD_SHARED_LIBS=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_OPENMP=OFF \
|
||||
-DGGML_WEBGPU=ON \
|
||||
-DDawn_DIR=$PWD/wasi-webgpu \
|
||||
-DLLAMA_BUILD_COMMON=OFF
|
||||
|
||||
time cmake --build build-wasi --config Release --target llama -j $(nproc)
|
||||
|
||||
LIBS="build-wasi/ggml/src/libggml.a build-wasi/ggml/src/ggml-webgpu/libggml-webgpu.a build-wasi/ggml/src/libggml-cpu.a build-wasi/ggml/src/libggml-base.a"
|
||||
LDFLAGS="-lunwind -ldl -Wl,-z,stack-size=8388608"
|
||||
CXX="${WASI_SDK_PATH}/bin/clang++ --target=wasm32-wasip3 -O3 -std=c++17 ${WASI_FLAGS} -I include -I ggml/include"
|
||||
|
||||
$CXX examples/simple/simple.cpp build-wasi/src/libllama.a $LIBS $LDFLAGS -o build-wasi/llama-simple.wasm
|
||||
$CXX tests/test-backend-ops.cpp $LIBS $LDFLAGS -o build-wasi/test-backend-ops.wasm
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasi
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
@@ -50,7 +50,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ubuntu-webgpu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasm
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-26.04-arm-wasm
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -7,7 +7,15 @@ on:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-webgpu.yml',
|
||||
'ggml/src/ggml-webgpu/**'
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.wgsl',
|
||||
'**/*.tmpl',
|
||||
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
@@ -34,7 +42,7 @@ env:
|
||||
|
||||
jobs:
|
||||
format:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -45,7 +53,7 @@ jobs:
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key |
|
||||
sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
||||
sudo add-apt-repository -y \
|
||||
"deb http://apt.llvm.org/resolute/ llvm-toolchain-resolute-22 main"
|
||||
"deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang-format-22
|
||||
|
||||
@@ -55,3 +63,135 @@ jobs:
|
||||
-type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) \
|
||||
-print0 |
|
||||
xargs -0 clang-format-22 --dry-run --Werror
|
||||
|
||||
macos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-macos-latest
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
export CMAKE_PREFIX_PATH=dawn
|
||||
cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
|
||||
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-macos-latest
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
ubuntu:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo add-apt-repository -y ppa:kisak/kisak-mesa
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y build-essential mesa-vulkan-drivers \
|
||||
libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
export Dawn_DIR=dawn/lib64/cmake/Dawn
|
||||
cmake -B build \
|
||||
-DGGML_WEBGPU=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
@@ -46,54 +46,17 @@ env:
|
||||
|
||||
jobs:
|
||||
cpu-x64-high-perf:
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
# note: this runner has 8 CPUs, but std::thread::hardware_concurrency reports the cores of the
|
||||
# host (https://github.com/ggml-org/llama.cpp/pull/31439#issuecomment-6683386059)
|
||||
env:
|
||||
TEST_ARG_HARDWARE_CONCURRENCY: 8
|
||||
TEST_ARG_PARALLEL_WORKERS: 4
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libssl-dev time unzip wget python3 python3-venv python3-dev
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-cpu-x64-high-perf
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-cpu-x64-high-perf
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-arm64-high-perf-graviton4:
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
@@ -93,72 +93,13 @@ jobs:
|
||||
save: true
|
||||
|
||||
gpu-rocm:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Strix Halo (iGPU, RDNA3.5)
|
||||
- gpu: gfx1151
|
||||
runner: [self-hosted, Linux, strix-halo, devlab-dispatch]
|
||||
# async-execution correctness workaround, gfx1151 only (see Test step)
|
||||
hip_launch_blocking: "1"
|
||||
# Radeon AI PRO R9700 (dGPU, RDNA4)
|
||||
- gpu: gfx1201
|
||||
runner: [self-hosted, Linux, radeon-r9700, devlab-dispatch]
|
||||
hip_launch_blocking: "0"
|
||||
# HIP-graph capture fails on gfx1201 (hipMemset -> hipErrorStreamCaptureImplicit)
|
||||
disable_hip_graphs: true
|
||||
|
||||
name: gpu-rocm (${{ matrix.gpu }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 120
|
||||
|
||||
env:
|
||||
ROCM_VERSION: "10.0.0"
|
||||
runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Verify GPU device
|
||||
run: |
|
||||
ls /dev/kfd /dev/dri/renderD* || { echo "ERROR: no amdgpu compute device"; exit 1; }
|
||||
|
||||
# same TheRock wheels as the ubuntu-24-rocm release build, so the runner does not need a ROCm install
|
||||
- name: Setup TheRock with Wheels
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/rocm"
|
||||
"$RUNNER_TEMP/rocm/bin/python" -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel,device-${{ matrix.gpu }}]==${ROCM_VERSION}"
|
||||
ROCM_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --root)
|
||||
{
|
||||
echo "ROCM_PATH=$ROCM_PATH"
|
||||
echo "HIP_PATH=$ROCM_PATH"
|
||||
echo "CMAKE_PREFIX_PATH=$("$RUNNER_TEMP/rocm/bin/rocm-sdk" path --cmake)"
|
||||
echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}"
|
||||
} >> "$GITHUB_ENV"
|
||||
"$RUNNER_TEMP/rocm/bin/rocm-sdk" path --bin >> "$GITHUB_PATH"
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
# ROCm is installed again on each run, so hash the compiler content, not its mtime
|
||||
- name: Tune ccache for reinstalled ROCm toolchain
|
||||
run: |
|
||||
ccache --set-config=compiler_check=content
|
||||
ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
||||
@@ -166,25 +107,10 @@ jobs:
|
||||
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
||||
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
||||
env:
|
||||
HIP_LAUNCH_BLOCKING: ${{ matrix.hip_launch_blocking }}
|
||||
HIP_LAUNCH_BLOCKING: "1"
|
||||
run: |
|
||||
rocminfo
|
||||
# Exported rather than set in env:, because ggml only checks that the
|
||||
# variable is defined: an empty value would disable graphs on gfx1151 too.
|
||||
if [ "${{ matrix.disable_hip_graphs }}" = "true" ]; then export GGML_CUDA_DISABLE_GRAPHS=1; fi
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=${{ matrix.gpu }} bash ./ci/run.sh "$RUNNER_TEMP/results/llama.cpp" "$RUNNER_TEMP/mnt/llama.cpp"
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-gpu-rocm-${{ matrix.gpu }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-rocm:
|
||||
|
||||
@@ -60,4 +60,4 @@ jobs:
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -21,9 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-openvino.yml',
|
||||
'ci/run.sh',
|
||||
'CMakeLists.txt',
|
||||
'ggml/CMakeLists.txt',
|
||||
'tests/CMakeLists.txt',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
@@ -52,7 +50,7 @@ jobs:
|
||||
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# - name: ccache-buckets-save
|
||||
# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# - name: ccache-buckets-save
|
||||
# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-intel-linux:
|
||||
runs-on: [self-hosted, Linux, Intel]
|
||||
|
||||
@@ -131,4 +131,4 @@ jobs:
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
||||
bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -16,7 +16,7 @@ cache-mode: none
|
||||
jobs:
|
||||
# The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
|
||||
copilot-setup-steps:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
# Set the permissions to the lowest permissions possible needed for your steps.
|
||||
# Copilot will be given its own token for its operations.
|
||||
|
||||
@@ -16,10 +16,6 @@ on:
|
||||
description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)"
|
||||
type: boolean
|
||||
default: false
|
||||
skip_ref_check:
|
||||
description: "Skip the requirement to run from a tag (allows branch refs)"
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
@@ -42,14 +38,6 @@ jobs:
|
||||
source_tag: ${{ steps.srctag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: Check manual run ref
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_ref_check }}
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then
|
||||
echo "::error::Manual runs of this workflow must be started from a tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -82,7 +70,7 @@ jobs:
|
||||
|
||||
prepare_matrices:
|
||||
name: Prepare Docker matrices
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
build_matrix: ${{ steps.matrices.outputs.build_matrix }}
|
||||
merge_matrix: ${{ steps.matrices.outputs.merge_matrix }}
|
||||
@@ -99,19 +87,19 @@ jobs:
|
||||
# Keep all build targets in one place and derive merge targets from it.
|
||||
cat > build-matrix.json <<'JSON'
|
||||
[
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-s390x", "prebuilt_ui": true },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" },
|
||||
{ "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" },
|
||||
{ "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" }
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }
|
||||
]
|
||||
JSON
|
||||
|
||||
@@ -120,9 +108,6 @@ jobs:
|
||||
mv build-matrix.json.tmp build-matrix.json
|
||||
fi
|
||||
|
||||
jq 'map(select(.tag != "musa"))' build-matrix.json > build-matrix.json.tmp
|
||||
mv build-matrix.json.tmp build-matrix.json
|
||||
|
||||
BUILD_MATRIX="$(jq -c . build-matrix.json)"
|
||||
MERGE_MATRIX="$(jq -c '
|
||||
reduce .[] as $entry ({}; .[$entry.tag] |= (
|
||||
@@ -185,7 +170,7 @@ jobs:
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
@@ -415,7 +400,7 @@ jobs:
|
||||
merge_arch_tags:
|
||||
name: Create shared tags from digests
|
||||
needs: [prepare_matrices, push_to_registry, create_tag]
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
@@ -24,7 +24,7 @@ permissions:
|
||||
jobs:
|
||||
deploy:
|
||||
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -39,7 +39,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
make-release:
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -61,17 +61,8 @@ jobs:
|
||||
RELEASE_BRANCH: ${{ github.ref_name }}
|
||||
SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }}
|
||||
APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }}
|
||||
APIABI_REPORT_DIR: ${{ runner.temp }}/apiabi-reports
|
||||
REQUIRE_DOCKER: ${{ github.event.inputs.require_docker }}
|
||||
|
||||
- name: Upload API/ABI reports
|
||||
if: ${{ always() && github.event.inputs.skip_apiabi_check != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: apiabi-reports
|
||||
path: ${{ runner.temp }}/apiabi-reports
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Create release tag
|
||||
if: ${{ github.event.inputs.dry_run == 'false' }}
|
||||
run: |
|
||||
|
||||
@@ -11,8 +11,7 @@ on:
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/llama-graph*',
|
||||
'src/llama-kv-cache*',
|
||||
'src/llama-graph.cpp',
|
||||
'src/llama-model*',
|
||||
'src/models/**'
|
||||
]
|
||||
@@ -25,8 +24,7 @@ on:
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/llama-graph*',
|
||||
'src/llama-kv-cache*',
|
||||
'src/llama-graph.cpp',
|
||||
'src/llama-model*',
|
||||
'src/models/**'
|
||||
]
|
||||
@@ -147,11 +145,6 @@ jobs:
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
- name: Test KV shift
|
||||
id: test_kv_shift
|
||||
run: |
|
||||
./build/bin/test-llama-archs --kv-shift
|
||||
|
||||
- name: Test fusion
|
||||
id: test_fusion
|
||||
run: |
|
||||
@@ -300,11 +293,6 @@ jobs:
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
|
||||
|
||||
- name: Test KV shift
|
||||
id: test_kv_shift
|
||||
run: |
|
||||
./build/bin/test-llama-archs --kv-shift
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
name: "PR video demo label"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, reopened, synchronize]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
video-demo-label:
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Add or remove the video demo label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
# passed through env, so the body is only matched, never run, and is safe to read from a fork
|
||||
BODY: ${{ github.event.pull_request.body }}
|
||||
HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
|
||||
run: |
|
||||
# sections written by demo:pr (see ui/apps/chat/demo/cli.ts): the managed comment
|
||||
# markers, or a preview linking to a demo video uploaded for this PR
|
||||
has_demo=false
|
||||
if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
|
||||
grep -qE "/resolve/videos/pr/$PR_NUMBER/[0-9a-f]{10}(-[0-9]+)?\.mp4" <<< "$BODY"; then
|
||||
has_demo=true
|
||||
fi
|
||||
|
||||
if [ "$has_demo" = true ] && [ "$HAS_LABEL" = false ]; then
|
||||
gh pr edit "$PR_URL" --add-label 'video demo'
|
||||
elif [ "$has_demo" = false ] && [ "$HAS_LABEL" = true ]; then
|
||||
gh pr edit "$PR_URL" --remove-label 'video demo'
|
||||
fi
|
||||
@@ -24,7 +24,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
python-check-requirements:
|
||||
runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }}
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
name: check-requirements
|
||||
steps:
|
||||
- name: Check out source repository
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.100
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.84
|
||||
# - name: Type-check with Pyright
|
||||
# uses: jakebailey/pyright-action@v2
|
||||
# with:
|
||||
|
||||
@@ -9,8 +9,6 @@ on:
|
||||
branches:
|
||||
- master
|
||||
|
||||
run-name: "Publish ${{ github.event.workflow_run.display_title }}"
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: read
|
||||
@@ -171,16 +169,14 @@ jobs:
|
||||
**Linux:**
|
||||
- [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
|
||||
- [Ubuntu 24.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz)
|
||||
- [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz)
|
||||
- [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
|
||||
- [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
|
||||
- [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
|
||||
- [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-2026.4.1-x64.tar.gz)
|
||||
- [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
|
||||
- [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
|
||||
- [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
|
||||
- [Linux arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md)
|
||||
@@ -194,11 +190,11 @@ jobs:
|
||||
- [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
|
||||
- [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
|
||||
- [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-x64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-x64.zip)
|
||||
- [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-arm64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-arm64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
|
||||
- [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
|
||||
- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
|
||||
- [Windows arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-arm64.zip)
|
||||
- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-2026.4.1-x64.zip)
|
||||
- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
|
||||
- [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
|
||||
- [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)
|
||||
|
||||
|
||||
+71
-149
@@ -35,7 +35,7 @@ permissions:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
|
||||
CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DLLAMA_CONNECT=ON"
|
||||
CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"
|
||||
|
||||
# note: run this workflow one at a time for better cache reuse
|
||||
concurrency:
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -180,8 +180,6 @@ jobs:
|
||||
os: ubuntu-24.04-arm
|
||||
- build: 's390x'
|
||||
os: ubuntu-24.04-s390x
|
||||
- build: '26.04-s390x'
|
||||
os: ubuntu-26.04-s390x
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@@ -196,13 +194,13 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libssl-dev
|
||||
sudo apt-get install build-essential libssl-dev
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
|
||||
@@ -212,7 +210,7 @@ jobs:
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: ccache
|
||||
if: ${{ !contains(matrix.os, 's390x') }}
|
||||
if: ${{ matrix.build != 's390x' }}
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
key: release-${{ matrix.os }}-cpu
|
||||
@@ -244,7 +242,7 @@ jobs:
|
||||
archive: false
|
||||
|
||||
- name: ccache-clear
|
||||
if: ${{ !contains(matrix.os, 's390x') }}
|
||||
if: ${{ matrix.build != 's390x' }}
|
||||
uses: ./.github/actions/ccache-clear
|
||||
with:
|
||||
key: release-${{ matrix.os }}-cpu
|
||||
@@ -274,7 +272,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -345,14 +343,14 @@ jobs:
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- build: 'x64'
|
||||
os: ubuntu-24.04
|
||||
cuda: '13.5.0'
|
||||
label: '13.5'
|
||||
defines: ''
|
||||
cuda: '13.4.1'
|
||||
label: '13.4'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- build: 'arm64'
|
||||
os: ubuntu-24.04-arm
|
||||
cuda: '13.5.0'
|
||||
label: '13.5'
|
||||
defines: ''
|
||||
cuda: '13.4.1'
|
||||
label: '13.4'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04
|
||||
@@ -381,7 +379,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -463,7 +461,7 @@ jobs:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
|
||||
env:
|
||||
NDK_VERSION: "29.0.14206865"
|
||||
@@ -479,7 +477,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
@@ -548,8 +546,8 @@ jobs:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8'
|
||||
runs-on: ubuntu-latest
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@@ -571,7 +569,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -601,8 +599,8 @@ jobs:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8'
|
||||
runs-on: ubuntu-latest
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@@ -624,7 +622,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -656,12 +654,19 @@ jobs:
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
outputs:
|
||||
openvino_version: ${{ steps.openvino_version.outputs.value }}
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
steps:
|
||||
- name: Set OpenVINO version output
|
||||
id: openvino_version
|
||||
run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -672,7 +677,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -716,9 +721,6 @@ jobs:
|
||||
-DGGML_OPENVINO=ON \
|
||||
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
|
||||
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
|
||||
-DGGML_BACKEND_DL=ON \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_CPU_ALL_VARIANTS=ON \
|
||||
${{ env.CMAKE_ARGS }}
|
||||
cmake --build build/ReleaseOV --config Release --parallel
|
||||
|
||||
@@ -757,26 +759,38 @@ jobs:
|
||||
with:
|
||||
key: release-ubuntu-24.04-openvino-release-no-preset-v1
|
||||
|
||||
# note: builds only the ggml-openvino backend - llama-server and the CPU backend are injected from the
|
||||
# windows-cpu zip during the release "Merge artifacts" step
|
||||
windows-openvino:
|
||||
needs: [check-release]
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: windows-2022
|
||||
|
||||
outputs:
|
||||
openvino_version: ${{ steps.openvino_version.outputs.value }}
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
steps:
|
||||
- name: Set OpenVINO version output
|
||||
id: openvino_version
|
||||
shell: bash
|
||||
run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download UI build
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -824,13 +838,11 @@ jobs:
|
||||
-A x64 ^
|
||||
-DCMAKE_BUILD_TYPE=Release ^
|
||||
-DGGML_OPENVINO=ON ^
|
||||
-DGGML_BACKEND_DL=ON ^
|
||||
-DGGML_NATIVE=OFF ^
|
||||
-DGGML_CPU=OFF ^
|
||||
-DLLAMA_BUILD_BORINGSSL=ON ^
|
||||
-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
|
||||
-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
|
||||
${{ env.CMAKE_ARGS }}
|
||||
|
||||
cmake --build build\ReleaseOV --config Release --target ggml-openvino -- /m
|
||||
cmake --build build\ReleaseOV --config Release -- /m
|
||||
|
||||
- name: Pack artifacts
|
||||
id: pack_artifacts
|
||||
@@ -858,12 +870,12 @@ jobs:
|
||||
Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force
|
||||
|
||||
Copy-Item LICENSE $dest
|
||||
7z a -snl llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\*
|
||||
7z a -snl llama-${{ needs.check-release.outputs.tag_name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\*
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
path: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
|
||||
path: llama-${{ needs.check-release.outputs.tag_name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
|
||||
archive: false
|
||||
|
||||
- name: ccache-clear
|
||||
@@ -894,17 +906,12 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Install Ninja
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -975,11 +982,6 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
max-size: "1G"
|
||||
|
||||
- name: Tune ccache for ROCm
|
||||
run: |
|
||||
# since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
# - name: Cache ROCm Installation
|
||||
# id: cache-rocm
|
||||
# uses: actions/cache@v5
|
||||
@@ -1151,11 +1153,6 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
# TODO: these jobs need to use llvm toolchain in order to utilize the ccache
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -1221,12 +1218,12 @@ jobs:
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.5'
|
||||
- cuda: '13.4'
|
||||
arch: x64
|
||||
defines: ''
|
||||
- cuda: '13.5'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.4'
|
||||
arch: arm64
|
||||
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -1249,7 +1246,6 @@ jobs:
|
||||
with:
|
||||
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
|
||||
evict-old-files: 1d
|
||||
max-size: "1G"
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -1318,14 +1314,11 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b177a006-5cd9-4cb2-807f-8199cc73d53c/intel-deep-learning-essentials-2026.2.0.638_offline.exe
|
||||
WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/8f905666-c85c-43fb-9bd0-2618c7ecd33b/intel-onednn-2026.1.0.622_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
|
||||
WINDOWS_DNNL: intel.oneapi.win.dnnl
|
||||
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.34.0/level-zero-win-sdk-1.34.0.zip
|
||||
WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -1335,8 +1328,7 @@ jobs:
|
||||
- name: Download & Install oneAPI
|
||||
shell: bash
|
||||
run: |
|
||||
scripts/install-oneapi.bat $WINDOWS_DLE_URL $WINDOWS_DPCPP_MKL
|
||||
scripts/install-oneapi.bat $WINDOWS_ONEDNN_URL $WINDOWS_DNNL
|
||||
scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
|
||||
|
||||
- name: Install Level Zero SDK
|
||||
shell: pwsh
|
||||
@@ -1374,12 +1366,11 @@ jobs:
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_def.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_avx2.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_intel_thread.3.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/libimalloc.dll" ./build/bin
|
||||
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader_shared.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin
|
||||
cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin
|
||||
ZE_LOADER_DLL=$(find "${{ env.ONEAPI_ROOT }}" "$LEVEL_ZERO_V1_SDK_PATH" -iname ze_loader.dll -print -quit 2>/dev/null || true)
|
||||
if [ -n "$ZE_LOADER_DLL" ]; then
|
||||
@@ -1433,8 +1424,8 @@ jobs:
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.2"
|
||||
LEVEL_ZERO_VERSION: "1.34.0"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
steps:
|
||||
@@ -1448,15 +1439,8 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cd /tmp
|
||||
echo "Download oneAPI Deep Learning Essentials installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/aef232d2-aac5-4cd6-bceb-911019bbb6a8/intel-deep-learning-essentials-2026.2.0.640_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
echo "Install oneAPI Deep Learning Essentials components"
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
echo "Download oneDNN installation package"
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/46452a4a-7b7f-49f2-be4c-602a900180bb/intel-onednn-2026.1.0.625_offline.sh -O intel-onednn_offline.sh
|
||||
echo "Install oneDNN components"
|
||||
sudo bash intel-onednn_offline.sh -s -a --silent --eula accept
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
|
||||
sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
|
||||
|
||||
- name: Install Level Zero SDK
|
||||
shell: bash
|
||||
@@ -1470,13 +1454,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libssl-dev
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -1496,8 +1474,8 @@ jobs:
|
||||
-DCMAKE_CXX_COMPILER=icpx \
|
||||
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
|
||||
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DLLAMA_CONNECT=ON \
|
||||
-DGGML_SYCL_F16=${{ matrix.fp16 }}
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
@@ -1542,7 +1520,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Free up disk space
|
||||
uses: ggml-org/free-disk-space@v1.3.1
|
||||
@@ -1564,8 +1542,6 @@ jobs:
|
||||
# builds hit.
|
||||
ccache --set-config=compiler_check=content
|
||||
ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
|
||||
# since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
|
||||
ccache --set-config=depend_mode=true
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1640,60 +1616,6 @@ jobs:
|
||||
with:
|
||||
key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
|
||||
|
||||
ubuntu-26-zdnn:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download UI build
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential libssl-dev libzdnn-dev
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
|
||||
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
|
||||
-DGGML_BACKEND_DL=ON \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_VXE=ON \
|
||||
-DGGML_ZDNN=ON \
|
||||
-DCMAKE_C_FLAGS="-march=arch15" \
|
||||
-DCMAKE_CXX_FLAGS="-march=arch15" \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
${{ env.CMAKE_ARGS }}
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Pack artifacts
|
||||
id: pack_artifacts
|
||||
run: |
|
||||
cp LICENSE ./build/bin/
|
||||
tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz
|
||||
archive: false
|
||||
|
||||
ios-xcode:
|
||||
needs: [check-release]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
@@ -42,8 +42,6 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
LLAMA_ARG_LOG_VERBOSITY: 10
|
||||
# [TAG_MTMD_NO_REALLOC] must match -DGGML_SCHED_NO_REALLOC=ON below, skips tests that are known to realloc
|
||||
GGML_SCHED_NO_REALLOC: ON
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
|
||||
@@ -39,8 +39,6 @@ env:
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
LLAMA_ARG_LOG_VERBOSITY: 10
|
||||
# [TAG_MTMD_NO_REALLOC] must match -DGGML_SCHED_NO_REALLOC=ON below, skips tests that are known to realloc
|
||||
GGML_SCHED_NO_REALLOC: ON
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
name: Server (speculative decoding)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
spec_methods:
|
||||
description: 'Speculative methods to test'
|
||||
required: true
|
||||
default: 'compact'
|
||||
type: choice
|
||||
options:
|
||||
- compact
|
||||
- nightly
|
||||
- all
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
server-speculative:
|
||||
runs-on: [self-hosted, Linux, gb10, perf]
|
||||
container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y cmake libssl-dev python3 python3-venv python3-pip
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: Python setup
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: ${{ github.event.inputs.spec_methods || 'compact' }}
|
||||
# pull requests from forks do not get repository variables
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || '/mnt/cache/spec-models' }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
|
||||
@@ -59,7 +59,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
ubuntu:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Dependencies
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: server-ubuntu-26.04-arm
|
||||
key: server-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: server-ubuntu-26.04-arm
|
||||
key: server-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
@@ -175,11 +175,6 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -23,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Build application
|
||||
run: npm run build
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
retention-days: 1
|
||||
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
runs-on: ubuntu-slim
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -37,26 +37,26 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Build application
|
||||
env:
|
||||
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
|
||||
run: npm run build
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run PWA unit tests (versioned build output)
|
||||
run: npx vitest --project=unit --run tests/unit/pwa.spec.ts
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
retention-days: 1
|
||||
@@ -1,183 +0,0 @@
|
||||
name: UI Chat Publish
|
||||
|
||||
# Publishes Llama Chat to npm: @ggml-org/llama-chat (ui/apps/chat, the app and its CLI) and
|
||||
# llama-chat (ui/apps/chat/alias, the short name for npx), always at the same version.
|
||||
#
|
||||
# The version in ui/apps/chat/package.json is the release, set by hand, as for @ggml-org/llama-ui
|
||||
# in ui-lib-publish.yml. A commit that reaches master with a version npm does not have builds the
|
||||
# app and publishes both packages; any other commit publishes nothing. Bump both package.json
|
||||
# files in the PR that should ship.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarballs with provenance. One-time setup, before this can publish: on
|
||||
# each of the two packages, Publishing settings, Trusted Publisher, GitHub Actions, with
|
||||
# organization ggml-org, repository llama.cpp, workflow file ui-chat-publish.yml and Environment
|
||||
# left blank. Until then the publish step fails on master; re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/package.json'
|
||||
]
|
||||
# a dry run for PRs that change what goes into the packages
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/**',
|
||||
'ui/apps/chat/bin/**',
|
||||
'ui/apps/chat/cli/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
APP_DIR: ui/apps/chat
|
||||
ALIAS_DIR: ui/apps/chat/alias
|
||||
VERSION_CHECK: ui/lib/ui/scripts/npm-version-check.mjs
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-chat and llama-chat
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Build the app
|
||||
# the same build.json version as the UI that llama-server embeds, for the app's update check.
|
||||
# This folder's .npmrc sets ignore-scripts, so npm never builds it: this step does.
|
||||
run: LLAMA_BUILD_NUMBER="b$(git rev-list --count HEAD)" npm run build
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Test the CLI and the alias version
|
||||
run: npx vitest --project=unit --run tests/unit/cli.test.ts
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Check what goes into the package
|
||||
run: |
|
||||
npm pack --dry-run --json > "$RUNNER_TEMP/pack.json"
|
||||
node -e '
|
||||
const [{ files }] = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
||||
const paths = files.map((file) => file.path);
|
||||
const maps = paths.filter((path) => path.endsWith(".map"));
|
||||
if (!paths.includes("dist/index.html")) {
|
||||
console.error("dist/index.html is not in the package: nothing would be served");
|
||||
process.exit(1);
|
||||
}
|
||||
if (maps.length > 0) {
|
||||
console.error(`source maps in the package: ${maps.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${paths.length} files, no source maps`);
|
||||
' "$RUNNER_TEMP/pack.json"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Compare @ggml-org/llama-chat with npm
|
||||
id: app
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$APP_DIR"
|
||||
|
||||
- name: Compare llama-chat with npm
|
||||
id: alias
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$ALIAS_DIR"
|
||||
|
||||
# master only: npm's trusted publisher matches the repository and the workflow file, not the
|
||||
# branch, so this condition is what keeps a manual run on a PR branch from publishing.
|
||||
# The app goes first, since the alias depends on it at the same version.
|
||||
- name: Publish @ggml-org/llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.app.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.app.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Publish llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.ALIAS_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# values reach the shell through env, so a name from a PR's package.json cannot inject here
|
||||
env:
|
||||
APP: ${{ steps.app.outputs.name }}@${{ steps.app.outputs.version }}
|
||||
APP_VERDICT: ${{ steps.app.outputs.verdict || 'not reached' }}
|
||||
APP_NOTE: ${{ steps.app.outputs.note }}
|
||||
ALIAS: ${{ steps.alias.outputs.name }}@${{ steps.alias.outputs.version }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
{
|
||||
echo "### Llama Chat on npm"
|
||||
echo
|
||||
echo "| package | verdict | note |"
|
||||
echo "| --- | --- | --- |"
|
||||
echo "| \`$APP\` | \`$APP_VERDICT\` | $APP_NOTE |"
|
||||
echo "| \`$ALIAS\` | \`$ALIAS_VERDICT\` | $ALIAS_NOTE |"
|
||||
echo
|
||||
if [ "$EVENT" = "pull_request" ] || [ "$DRY" = "true" ]; then echo "Dry run: nothing was published."; fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
# a version going backwards is a mistake; an unchanged version is a package.json edit that
|
||||
# is not a release, such as a dependency update
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
env:
|
||||
VERDICTS: ${{ steps.app.outputs.verdict }} ${{ steps.alias.outputs.verdict }}
|
||||
run: |
|
||||
case " $VERDICTS " in
|
||||
*" stale "*)
|
||||
echo "::error::a version in package.json is older than the one on npm"
|
||||
exit 1 ;;
|
||||
esac
|
||||
@@ -1,230 +0,0 @@
|
||||
name: UI Lib Publish
|
||||
|
||||
# Publishes @ggml-org/llama-ui, the library in ui/lib/ui, to npm.
|
||||
#
|
||||
# The version in ui/lib/ui/package.json is the release. A commit that reaches master with a
|
||||
# version npm does not have builds the library and publishes it; a commit that changes the
|
||||
# library without bumping it publishes nothing and says so on the run, which is the drift that
|
||||
# let 0.3.0 sit a day behind master. Bump the version in the PR whose change should ship, and
|
||||
# this workflow does the rest.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarball with a provenance attestation. release-publish.yml has done
|
||||
# the same exchange on every push to master since it landed, its actions/attest step being the
|
||||
# proof that this repository can be issued a token.
|
||||
#
|
||||
# One-time setup, before this can publish: on the package, Publishing settings, Trusted
|
||||
# Publisher, GitHub Actions, with organization ggml-org, repository llama.cpp, workflow file
|
||||
# ui-lib-publish.yml, and Environment left blank - this job names no environment, since creating
|
||||
# one needs repository admin. Merging this workflow before that exists leaves the publish step
|
||||
# red on master: the release is late rather than wrong, and re-running the job publishes it.
|
||||
#
|
||||
# The unscoped llama-ui is an alias: once the scoped package is out, the same build is renamed
|
||||
# and published again at the same version, so the two never drift. It is a package of its own
|
||||
# on npm, and needs the same trusted publisher set on it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
# the advisory half, so a library change without a bump is annotated on the PR rather than
|
||||
# discovered after the merge: the path filter keeps it off PRs that touch only Llama Chat
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back. GitHub holds
|
||||
# one pending run per group, so a newer push replaces an older queued one - no release is lost,
|
||||
# as each run reads the version from its own commit.
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
LIB_DIR: ui/lib/ui
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-ui
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
npm -v
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Build the package and lint it
|
||||
run: npm run prepack
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the version with npm
|
||||
id: version
|
||||
# on a PR this is advisory and must not fail the job: a registry outage is not the
|
||||
# contributor's mistake. On master the publish decision rests on the answer, so it fails.
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Note a library change that ships no new version
|
||||
if: github.event_name == 'pull_request' && always() && steps.version.outcome == 'success' && steps.version.outputs.should_publish != 'true'
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
run: echo "::warning::$NOTE"
|
||||
|
||||
- name: Publish
|
||||
# master only. npm's trusted publisher matches on the repository and the workflow file,
|
||||
# not the branch, so this condition is the only thing stopping a manual run from a PR
|
||||
# branch - one that anyone with write access can start - from publishing code that no
|
||||
# one reviewed. A dispatch on master remains the way to re-run a release.
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
# --ignore-scripts publishes the tarball prepack built and publint passed rather than
|
||||
# rebuilding it here; --provenance is implied by trusted publishing and named to be
|
||||
# explicit. A package published with nothing in it cannot be taken back, so refuse first.
|
||||
env:
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Rename the build to the unscoped alias
|
||||
# after the scoped publish, so nothing above sees the changed name. Not on a PR: the
|
||||
# advisory check there is about the version, which the alias shares.
|
||||
if: github.event_name != 'pull_request'
|
||||
run: node scripts/alias-package.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the alias with npm
|
||||
id: alias
|
||||
if: github.event_name != 'pull_request'
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Publish the alias
|
||||
# its own verdict rather than the scoped one's, so an alias that fell behind catches up
|
||||
# on the next run instead of waiting for a bump
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# every value reaches the shell through env, not through the script, so a package name
|
||||
# from a PR's package.json cannot inject into this step
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name || '@ggml-org/llama-ui' }}
|
||||
VERSION: ${{ steps.version.outputs.version || 'unknown' }}
|
||||
LATEST: ${{ steps.version.outputs.latest }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict || 'not reached' }}
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
SHOULD: ${{ steps.version.outputs.should_publish }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
ALIAS_LATEST: ${{ steps.alias.outputs.latest }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
ALIAS_SHOULD: ${{ steps.alias.outputs.should_publish }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo "### $NAME $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
[ "$EVENT" = "pull_request" ] && exit 0
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$ALIAS_SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo
|
||||
echo "### llama-ui (alias) $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${ALIAS_LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$ALIAS_VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $ALIAS_NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
if: always() && github.event_name != 'pull_request' && steps.version.outcome == 'success'
|
||||
# a version going backwards is a mistake; a version not bumped is usually one too, but it
|
||||
# is also what a stories-only commit looks like, so that only fails the run when the
|
||||
# repository variable UI_LIB_PUBLISH_FAIL_ON_UNBUMPED is set to true
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict }}
|
||||
FAIL_ON_UNBUMPED: ${{ vars.UI_LIB_PUBLISH_FAIL_ON_UNBUMPED }}
|
||||
run: |
|
||||
case "$VERDICT" in
|
||||
stale)
|
||||
echo "::error::$NOTE"
|
||||
exit 1 ;;
|
||||
published)
|
||||
if [ "$FAIL_ON_UNBUMPED" = "true" ]; then
|
||||
echo "::error::$NOTE"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::$NOTE" ;;
|
||||
esac
|
||||
@@ -33,15 +33,15 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
tar -czf dist.tar.gz -C ui/apps/chat/dist .
|
||||
tar -czf dist.tar.gz -C tools/ui/dist .
|
||||
sha256sum dist.tar.gz > dist.tar.gz.sha256
|
||||
mv dist.tar.gz dist.tar.gz.sha256 ui/apps/chat/dist/
|
||||
mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: pip install -U huggingface_hub
|
||||
@@ -52,12 +52,12 @@ jobs:
|
||||
- name: Sync built files to Hugging Face bucket (version tag)
|
||||
run: |
|
||||
# Upload the built files to the Hugging Face bucket under the release version
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
|
||||
- name: Sync built files to Hugging Face bucket (latest)
|
||||
run: |
|
||||
# Also upload to the 'latest' directory for fallback downloads
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
|
||||
- name: Verify upload
|
||||
run: |
|
||||
|
||||
@@ -17,9 +17,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'ui/lib/voice/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -27,9 +25,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'ui/lib/voice/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -71,33 +67,33 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Unit tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -113,25 +109,25 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run E2E tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
@@ -1,84 +0,0 @@
|
||||
name: UI Storybook
|
||||
|
||||
# Builds Llama UI and Llama Chat's Storybook (ui/apps/chat) and publishes it to its Hugging Face Space
|
||||
# whenever the UI changes on master, so the Space always shows the latest build.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-storybook.yml',
|
||||
'ui/apps/chat/**',
|
||||
'ui/lib/ui/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# only the newest build matters: a newer push replaces a deploy still in progress
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to Hugging Face Spaces
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
env:
|
||||
HF_SPACE: ${{ vars.HF_SPACE_UI_STORYBOOK || 'ggml-org/llama-ui-storybook' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Write the Space card
|
||||
run: |
|
||||
cat > ui/apps/chat/storybook-static/README.md << EOF
|
||||
---
|
||||
title: Llama UI Storybook
|
||||
emoji: 🦙
|
||||
colorFrom: gray
|
||||
colorTo: gray
|
||||
sdk: static
|
||||
pinned: false
|
||||
---
|
||||
|
||||
The Storybook of Llama UI and Llama Chat, built from [ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp) (\`ui/apps/chat\`) at ${GITHUB_SHA}.
|
||||
EOF
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-venv
|
||||
python3 -m venv .venv-hf
|
||||
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
|
||||
|
||||
- name: Upload to the Space
|
||||
env:
|
||||
# a token with write access to the Space; HF_TOKEN_UI_STATIC_OUTPUT if that one has it
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STORYBOOK || secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
run: |
|
||||
# --delete '*' removes files the new build no longer has; the Space's .gitattributes is always kept
|
||||
.venv-hf/bin/hf upload "$HF_SPACE" ui/apps/chat/storybook-static . \
|
||||
--repo-type space \
|
||||
--delete '*' \
|
||||
--commit-message "Storybook from ggml-org/llama.cpp@${GITHUB_SHA::10}"
|
||||
+17
-91
@@ -13,8 +13,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -22,8 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -68,118 +66,46 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Check DESIGN.md is current with the tokens
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check:design
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run Unit tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
lib-checks:
|
||||
name: Library checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps chromium
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
# the unit tests and the Storybook stories (ui/lib/ui/tests/stories), which run as tests in Chromium
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
voice-checks:
|
||||
name: Voice checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/voice
|
||||
working-directory: tools/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -198,38 +124,38 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Download built UI artifacts (reuses ui-build)
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ui/apps/chat/dist/
|
||||
path: tools/ui/dist/
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Run E2E tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: ui/apps/chat
|
||||
working-directory: tools/ui
|
||||
@@ -12,7 +12,7 @@ permissions:
|
||||
jobs:
|
||||
update:
|
||||
name: Update Winget Package
|
||||
runs-on: ubuntu-26.04
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
if: github.repository_owner == 'ggml-org'
|
||||
|
||||
steps:
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
|
||||
Project-specific instructions:
|
||||
|
||||
## Coding
|
||||
|
||||
- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments
|
||||
@@ -0,0 +1,33 @@
|
||||
You are a coding agent. Here are some very important rules that you must follow:
|
||||
|
||||
General:
|
||||
- Be very precise and concise when writing code, comments, explanations, etc.
|
||||
- If an inline comment exceeds 2 lines, replace it with: `// note: TODO LATER`
|
||||
- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
|
||||
- Don't try to build or run the code unless you are explicitly asked to do so
|
||||
- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
|
||||
- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
|
||||
- Never read the `AGENTS.md` file
|
||||
|
||||
Coding:
|
||||
- When in doubt, always refer to the CONTRIBUTING.md file of the project
|
||||
- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments
|
||||
- When referencing issues or PRs in comments, use the format:
|
||||
- C/C++ code: `// ref: <url>`
|
||||
- Other (CMake, etc.): `# ref: <url>`
|
||||
|
||||
Pull requests (PRs):
|
||||
- New branch names are prefixed with "gg/"
|
||||
- Before opening a pull request, ask the user to confirm the description
|
||||
- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
|
||||
- When creating a pull request, look for the repository's PR template and follow it
|
||||
- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
|
||||
- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL]
|
||||
- Always create the pull requests in draft mode
|
||||
- Never reply to review comments or post comments on issues/PRs without explicit permission from the user
|
||||
|
||||
Commits:
|
||||
- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag
|
||||
- Do not explicitly set the git author in commits - rely on the default git config
|
||||
- Always use `--no-gpg-sign` when committing
|
||||
- Never `git push` without explicit confirmation from the user
|
||||
@@ -246,6 +246,3 @@ Chat template and parser:
|
||||
- [PEG parser](docs/development/parsing.md) - alternative to regex that llama.cpp uses to parse model's output
|
||||
- [Auto parser](docs/autoparser.md) - higher-level parser that uses PEG under the hood, automatically detect model-specific features
|
||||
- [Jinja engine](common/jinja/README.md)
|
||||
|
||||
Testing:
|
||||
- [Slice model](skills/slice-model/SKILL.md) - build a small slice (few layers/experts) of a big HF model without downloading the full weights, to test conversion and inference
|
||||
+4
-8
@@ -3,8 +3,8 @@ project("llama.cpp" C CXX)
|
||||
include(CheckIncludeFileCXX)
|
||||
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 1)
|
||||
set(LLAMA_VERSION_MINOR 0)
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 6)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
@@ -103,7 +103,7 @@ endif()
|
||||
|
||||
# subprocess spawning isn't a supported/sandbox-friendly operation on mobile OSes or in WASM
|
||||
if (CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android" OR ANDROID
|
||||
OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN OR CMAKE_SYSTEM_NAME STREQUAL "WASI")
|
||||
OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN)
|
||||
set(LLAMA_SUBPROCESS_DEFAULT OFF)
|
||||
else()
|
||||
set(LLAMA_SUBPROCESS_DEFAULT ON)
|
||||
@@ -134,9 +134,8 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
|
||||
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_UI "llama: build Llama Chat, the embedded chat app, for server" OFF)
|
||||
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
|
||||
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
|
||||
option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)
|
||||
|
||||
option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})
|
||||
option(LLAMA_TESTS_INSTALL "llama: install tests" ON)
|
||||
@@ -150,7 +149,6 @@ option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured
|
||||
# Required for relocatable CMake package
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/win-app.cmake)
|
||||
|
||||
if (NOT DEFINED LLAMA_BUILD_NUMBER)
|
||||
set(LLAMA_BUILD_NUMBER ${BUILD_NUMBER})
|
||||
@@ -270,8 +268,6 @@ if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
|
||||
add_subdirectory(tools/mtmd)
|
||||
endif()
|
||||
|
||||
llama_apply_app_manifests()
|
||||
|
||||
#
|
||||
# install
|
||||
#
|
||||
|
||||
+30
-25
@@ -2,29 +2,20 @@
|
||||
# multiple collaborators per item can be specified
|
||||
#
|
||||
# ggml-org/ci : CISC, danbev, ggerganov, netrunnereve, ngxson, taronaeo
|
||||
# ggml-org/ggml-base : aendk, fairydreaming, ggerganov
|
||||
# ggml-org/ggml-cann : hipudding
|
||||
# ggml-org/ggml-cpu : angt, chaxu01, ggerganov, max-krasnyansky, taronaeo
|
||||
# ggml-org/ggml-cuda : am17an, gaugarg-nv, IMbackK, JohannesGaessler, ORippler, thelittlefireman, ynankani
|
||||
# ggml-org/ggml-et : marty1885
|
||||
# ggml-org/ggml-cuda : JohannesGaessler, am17an, IMbackK, ORippler
|
||||
# ggml-org/ggml-hexagon : lhez, max-krasnyansky
|
||||
# ggml-org/ggml-hip : IMbackK, thelittlefireman
|
||||
# ggml-org/ggml-metal : forforever73, ggerganov, jhen0409, masterFoad
|
||||
# ggml-org/ggml-metal : ggerganov
|
||||
# ggml-org/ggml-opencl : lhez, max-krasnyansky
|
||||
# ggml-org/ggml-openvino : ravi9, wine99
|
||||
# ggml-org/ggml-riscv : xctan
|
||||
# ggml-org/ggml-rpc : am17an, rgerganov
|
||||
# ggml-org/ggml-sycl : arthw, Titaniumtown
|
||||
# ggml-org/ggml-rpc : rgerganov
|
||||
# ggml-org/ggml-sycl : arthw
|
||||
# ggml-org/ggml-vulkan : 0cc4m, jeffbolznv
|
||||
# ggml-org/ggml-webgpu : reeselevine, yomaytk
|
||||
# ggml-org/ggml-zdnn : AlekseiNikiforovIBM, Andreas-Krebbel, taronaeo
|
||||
# ggml-org/llama-base : am17an, CISC, ggerganov, ngxson, sanmai, ServeurpersoCom
|
||||
# ggml-org/llama-common : aldehir, angt, bartowski1182, danbev, ggerganov, ngxson, nikwen, pwilkin, ruixiang63, sanmai
|
||||
# ggml-org/llama-examples : ggerganov
|
||||
# ggml-org/llama-models : CISC, danbev, ggerganov, ngxson
|
||||
# ggml-org/ggml-zdnn : taronaeo
|
||||
# ggml-org/llama-common : ggerganov, aldehir, angt, danbev, ngxson, pwilkin
|
||||
# ggml-org/llama-mtmd : ngxson
|
||||
# ggml-org/llama-server : allozaur, angt, ggerganov, ngxson, ServeurpersoCom
|
||||
# ggml-org/llama-ui : allozaur
|
||||
# ggml-org/llama-server : ggerganov, ngxson, allozaur, angt, ServeurpersoCom
|
||||
# ggml-org/llama-ui : allozaur
|
||||
|
||||
/.devops/*.Dockerfile @ngxson
|
||||
/.github/actions/ @ggml-org/ci
|
||||
@@ -38,12 +29,28 @@
|
||||
/conversion/ @CISC
|
||||
/convert_*.py @CISC
|
||||
/docs/backend/snapdragon/ @ggml-org/ggml-hexagon
|
||||
/examples/ @ggml-org/llama-examples
|
||||
/examples/batched.swift/ @ggerganov
|
||||
/examples/batched/ @ggerganov
|
||||
/examples/convert-llama2c-to-ggml/ @ggerganov
|
||||
/examples/debug/ @danbev @pwilkin
|
||||
/examples/deprecation-warning/ @ggerganov
|
||||
/examples/diffusion/ @am17an
|
||||
/examples/embedding/ @ggerganov
|
||||
/examples/eval-callback/ @ggerganov
|
||||
/examples/export-docs/ @ggerganov
|
||||
/examples/gen-docs/ @ggerganov
|
||||
/examples/gguf/ @ggerganov
|
||||
/examples/llama.android/ @ggerganov @hanyin-arm @naco-siren
|
||||
/examples/llama.swiftui/ @ggerganov
|
||||
/examples/llama.vim @ggerganov
|
||||
/examples/lookahead/ @ggerganov
|
||||
/examples/lookup/ @JohannesGaessler
|
||||
/examples/model-conversion/ @danbev
|
||||
/examples/parallel/ @ggerganov
|
||||
/examples/passkey/ @ggerganov
|
||||
/examples/retrieval/ @ggerganov
|
||||
/examples/speculative-simple/ @ggerganov
|
||||
/examples/speculative/ @ggerganov
|
||||
/ggml/cmake/ @ggerganov
|
||||
/ggml/include/ @ggerganov
|
||||
/ggml/src/ggml-backend-meta.cpp @JohannesGaessler
|
||||
@@ -55,7 +62,7 @@
|
||||
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
|
||||
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
|
||||
/ggml/src/ggml-hexagon/ @ggml-org/ggml-hexagon
|
||||
/ggml/src/ggml-hip/ @ggml-org/ggml-hip
|
||||
/ggml/src/ggml-hip/ @IMbackK
|
||||
/ggml/src/ggml-et/ @marty1885
|
||||
/ggml/src/ggml-impl.h @ggerganov
|
||||
/ggml/src/ggml-metal/ @ggml-org/ggml-metal
|
||||
@@ -69,7 +76,7 @@
|
||||
/ggml/src/ggml-virtgpu/ @kpouget
|
||||
/ggml/src/ggml-vulkan/ @ggml-org/ggml-vulkan
|
||||
/ggml/src/ggml-webgpu/ @ggml-org/ggml-webgpu
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
|
||||
/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar
|
||||
/ggml/src/ggml.c @ggerganov
|
||||
/ggml/src/ggml.cpp @ggerganov
|
||||
@@ -80,18 +87,16 @@
|
||||
/scripts/get* @ggerganov
|
||||
/scripts/sync* @ggerganov
|
||||
/scripts/snapdragon/ @ggml-org/ggml-hexagon
|
||||
/src/ @ggml-org/llama-base
|
||||
/src/ @ggerganov
|
||||
/src/llama-adapter.* @CISC
|
||||
/src/llama-arch.* @CISC
|
||||
/src/llama-chat.* @ngxson
|
||||
/src/llama-graph.* @CISC
|
||||
/src/llama-model.* @CISC
|
||||
/src/llama-vocab.* @CISC
|
||||
/src/llama-moe-cache.* @am17an
|
||||
/src/models/ @ggml-org/llama-models
|
||||
/src/models/ @CISC
|
||||
/tests/ @ggerganov
|
||||
/tests/test-chat.* @pwilkin
|
||||
/tests/test-server-component.cpp @ggml-org/llama-server
|
||||
/tools/batched-bench/ @ggerganov
|
||||
/tools/cli/ @ngxson
|
||||
/tools/completion/ @ggerganov
|
||||
@@ -102,7 +107,7 @@
|
||||
/tools/rpc/ @ggml-org/ggml-rpc
|
||||
/tools/server/* @ggml-org/llama-server # no subdir
|
||||
/tools/server/tests/ @ggml-org/llama-server
|
||||
/ui/ @ggml-org/llama-ui
|
||||
/tools/ui/ @ggml-org/llama-ui
|
||||
/tools/tokenize/ @ggerganov
|
||||
/tools/tts/ @ggerganov
|
||||
/vendor/ @ggerganov
|
||||
|
||||
@@ -16,8 +16,6 @@ The project differentiates between 3 levels of contributors:
|
||||
>
|
||||
> Detailed information regarding permissible and restricted uses of AI can be found in the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
### Requirements
|
||||
|
||||
If AI is used to generate any portion of the code, contributors must adhere to the following requirements:
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
@@ -26,16 +24,6 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
### Checklist
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> **If you use AI, do all of the following:**
|
||||
>
|
||||
> 1. **Make sure [AGENTS.md](AGENTS.md) is loaded into your agent's context.** Skipping this step is one of the most common reasons PRs get rejected.
|
||||
> 2. **Search for related discussions, PRs and issues** to get the broader context before writing any code.
|
||||
> 3. **Run a local review pass before pushing anything**, using the [`code-review`](skills/code-review/SKILL.md) skill, once the change is implemented. Fix what it finds.
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
# Pull requests (for contributors & collaborators)
|
||||
|
||||
@@ -1,53 +1,100 @@
|
||||
# llama.cpp
|
||||
|
||||

|
||||

|
||||
|
||||
<div align="center">
|
||||
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v0)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[v1.0.0](https://huggingface.co/blog/ggml-org/llama-cpp-v1) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
## Description
|
||||
## This fork - Polaris / GCN tuning for large MoE models
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal setup and state-of-the-art performance on a wide range of hardware - locally and in the cloud.
|
||||
Changes and measurements for running large MoE models with their experts offloaded to system RAM
|
||||
(`--n-cpu-moe`) on an old GCN card. The two code changes below are auto-on, need no flag, and are
|
||||
**token-identical** to mainline. Everything else here is tuning guidance.
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
Findings on an **RX 580 8GB** (Polaris / GCN, PCIe 3.0 x16, no fp16, no matrix cores) with
|
||||
**Qwen3.5-35B-A3B Q4_K_M**, `-b 2048 -ub 2048`:
|
||||
|
||||
## This fork - RX 580 (Polaris / GCN) serving of large MoE models
|
||||
- **Flash-attention `mask_opt` is enabled for GCN large head sizes (this fork's own change).**
|
||||
Upstream disables it on GCN; it is a **lossless** win in high-context prefill - it skips
|
||||
fully-masked causal blocks and the per-block mask add on fully-visible ones, which is real work on
|
||||
a card whose attention is compute-bound (no matrix cores). Auto-on, no flag. On Qwen3.5-35B
|
||||
(head_dim 256): pp2048 **+8% @ 16k, +12% @ 32k**, growing with depth; perplexity bit-identical.
|
||||
- **`-b 2048 -ub 2048` is the biggest prefill lever** (the default `-ub 512` roughly halves pp).
|
||||
- **Tune `--n-cpu-moe` to context length.** Keep some expert layers resident in spare VRAM for short
|
||||
prompts (e.g. `ncmoe 28` on the 35B, ~+5% over all-host); at long context the KV cache needs that
|
||||
VRAM, so raise it (`ncmoe 40`, all experts on host). Keep flash attention on (`-fa 1`).
|
||||
|
||||
A private fork maintained for one machine: an AMD RX 580 8 GB (RADV Vulkan) serving Qwen3.6-35B-A3B,
|
||||
gemma-4-26B-A4B and smaller models under llama-swap, with the MoE experts in host RAM
|
||||
(`--n-cpu-moe`). **[FORK.md](FORK.md) is the source of truth**: every fork change with its
|
||||
invariants, rebase hazards and validation, the measured dead ends, the production configuration and
|
||||
the measurement traps. The rebase and deploy procedure is
|
||||
[docs/fork-rebase-runbook.md](docs/fork-rebase-runbook.md); `scripts/fork-validate.sh` validates a
|
||||
build against a reference; `scripts/rx580-bench/` is the benchmark harness.
|
||||
**Recommended RX 580 / Polaris serving command** (per model):
|
||||
|
||||
In short, the fork adds:
|
||||
```bash
|
||||
llama-server -hf <repo>:<quant> -ngl 99 --n-cpu-moe 40 -b 2048 -ub 2048 -fa 1
|
||||
```
|
||||
|
||||
- three Polaris prefill wins, on without a flag and token-identical to mainline: the MoE routing-ids
|
||||
readback skip in `sched_copy_experts`, flash-attention `mask_opt` for GCN at head size >= 256, and
|
||||
a tiled-transpose fast path for the delta-net conv-state concat;
|
||||
- opt-in VRAM time-sharing between several llama-server processes (`LLAMA_SLEEP_VRAM_ONLY`,
|
||||
`LLAMA_SLEEP_EVICT_KV`, `LLAMA_MMPROJ_ONDEMAND`, `LLAMA_VRAM_ARENA`): a model releases everything it
|
||||
holds in VRAM and gets it back without a restart, so its RAM prompt cache survives;
|
||||
- fixes to the server's sleep, wake and shutdown paths and to cold teardown that the sharing exposed;
|
||||
- validation-only env knobs (`GGML_VK_CONCAT_FASTPATH`, `LLAMA_MOE_IDS_SKIP`), unset in production.
|
||||
Lower `--n-cpu-moe` (e.g. 28) if the model plus your context budget leave spare VRAM; keep it high
|
||||
for long-context / agentic use. At long context the bottleneck is attention compute (GPU-bound), so
|
||||
`mask_opt` (above) is where the remaining prefill wins come from, not the MoE-transfer path.
|
||||
|
||||
Production settings for the 35B are in FORK.md section 5 (`-lm none -t 6 -ngl 99 --n-cpu-moe 40
|
||||
-b 2048 -ub 2048 -fa 1 --parallel 1`, ctx 131072). Keep all experts on the host (`--n-cpu-moe 40`)
|
||||
for long contexts: lower values are faster only at short context and run out of VRAM as the KV grows
|
||||
(FORK.md section 4).
|
||||
### Very large MoE (experts bigger than the GTT limit)
|
||||
|
||||
Measured on the same RX 580 with **Laguna-S-2.1 118B IQ2_M** (48 layers, 256 experts, 10 used,
|
||||
experts 30.7 GiB of a 34.7 GiB file, all host-resident at `--n-cpu-moe 48`). The advice above changes
|
||||
in this regime:
|
||||
|
||||
- **`-lm none` (formerly `--no-mmap`) stops being an option.** Its pinned host buffer is charged
|
||||
against the amdgpu GTT limit (~31.4 GiB here, about half of system RAM). At 30.7 GiB of experts
|
||||
the model no longer loads, and the allocation spike can OOM the box. Use mmap and accept the
|
||||
staging copy.
|
||||
- **The routing-ids readback is pure overhead at prefill batch sizes** and this fork now skips it.
|
||||
To decide which experts to upload, the scheduler's expert-copy callback
|
||||
(`llama_context::sched_copy_experts`) reads the ids back from the device that has just produced
|
||||
them, forcing a full pipeline flush once per MoE layer per eval. With 2048 tokens x 10 experts over
|
||||
256 experts every expert is used anyway. Skipping it is exact - `mul_mat_id` only reads the rows
|
||||
the ids point at, and with `--moe-cache-mib` the cached experts are still copied device-to-device.
|
||||
Auto-on at `4 * n_expert` ids and above; decode keeps the old path.
|
||||
**pp2048 +4.7% @ 32k depth, +0.9% @ 16k, tg +3%.**
|
||||
- **Where the time actually goes** (`GGML_VK_PERF_LOGGER=1`, depth 0, 22.65 s per 2048-token eval,
|
||||
17.18 s of it GPU-busy so ~24% is H2D stall): expert `MUL_MAT_ID` **52%**, attention projections
|
||||
**24%**, `FLASH_ATTN_EXT` **16%**, everything else 8%. The expert matmuls run at 1686-2139 GFLOP/s
|
||||
while dense `MUL_MAT` q5_K/q6_K in the same graph reaches 3096-3691 - **the single largest
|
||||
remaining opportunity on this hardware is closing that gap**, not the transfer path.
|
||||
- **Interleaved SWA keeps its own small KV cache**, so a sliding-window layer costs the same at any
|
||||
depth (`n_kv` pinned at `n_swa * n_seq_max + n_ubatch`). On this model 36 of 48 layers are O(1) in
|
||||
depth and the entire high-context slowdown comes from the 12 full-attention layers.
|
||||
- **`--parallel 1`** is worth setting for a solo large model: the server otherwise auto-selects 4
|
||||
slots, and the SWA cache is sized `n_swa * n_seq_max + n_ubatch`, so 4 slots cost 4096 cells
|
||||
instead of 2560. Measured **223 MiB of VRAM freed** at 64k context.
|
||||
|
||||
Dead ends measured on this hardware, recorded so they are not retried:
|
||||
|
||||
| Change | Result |
|
||||
| --- | ---: |
|
||||
| flash-attn `shmem_staging` enabled for GCN | **-6.7% @ 16k, -7.4% @ 32k** |
|
||||
| `-b 4096 -ub 4096` (to amortize the fixed per-eval expert upload) | flat (-1%) |
|
||||
| `--n-cpu-moe` 44 instead of 48 | +1.5%, but does not fit at 64k ctx |
|
||||
| `mask_opt` gate relaxed below head_dim 256 | -18.5% @ 16k |
|
||||
|
||||
`shmem_staging` looks like a certain win (without it each rowgroup re-reads the whole K/V block
|
||||
through a 16 KiB L1) but the `kvsh` stride of `D/4+1` dwords is 4 mod 32, which costs an 8-way LDS
|
||||
bank conflict on wave64 - that `+1` padding is tuned for warp32. `-ub 4096` fails because halving the
|
||||
number of expert uploads is exactly cancelled by intra-ubatch attention growing quadratically.
|
||||
|
||||
**Serving note that outweighs all of the above.** With a model this large, anything that restarts the
|
||||
process is far more expensive than any kernel win: the server's prompt cache is RAM-only with no disk
|
||||
backing, so a restart forces a full re-prefill of the conversation. If a model swapper can evict this
|
||||
model to run a small helper model (chat-title generation and the like), fix that first - keeping the
|
||||
process alive across a swap took a repeat turn from a 21,960 ms prefill down to 225 ms.
|
||||
|
||||
## Quick start
|
||||
|
||||
@@ -83,12 +130,50 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
<i>VLM session with <b>llama cli</b></i>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img width="1392" height="958" alt="Llama Chat, the built-in chat app, against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Llama Chat, the built-in chat app, against <b>llama serve</b></i>
|
||||
<img width="1392" height="958" alt="Built-in web UI against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Built-in web UI against <b>llama serve</b></i>
|
||||
</td>
|
||||
</tr>
|
||||
<table>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on
|
||||
a wide range of hardware - locally and in the cloud.
|
||||
|
||||
- Plain C/C++ implementation without any dependencies
|
||||
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
||||
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
||||
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
||||
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
||||
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
||||
- Vulkan and SYCL backend support
|
||||
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
||||
|
||||
The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Supported backends
|
||||
|
||||
| Backend | Target devices |
|
||||
| --- | --- |
|
||||
| [BLAS](docs/build.md#blas-build) | All |
|
||||
| [BLIS](docs/backend/BLIS.md) | All |
|
||||
| [CANN](docs/build.md#cann) | Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
||||
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
||||
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
||||
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
||||
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
||||
| [Vulkan](docs/build.md#vulkan) | GPU |
|
||||
| [WebGPU](docs/build.md#webgpu) | All |
|
||||
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
||||
|
||||
## Documentation
|
||||
|
||||
#### Tools
|
||||
@@ -96,7 +181,6 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
- [cli](tools/cli/README.md)
|
||||
- [completion](tools/completion/README.md)
|
||||
- [server](tools/server/README.md)
|
||||
- [Text-to-speech (TTS) models](docs/multimodal/tts.md)
|
||||
- [GBNF grammars](grammars/README.md)
|
||||
|
||||
#### Development
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ Only vulnerabilities that fall within these parts of the project are considered
|
||||
- `ggml/**/*`
|
||||
- `gguf-py/**/*`
|
||||
- `tools/server/*`, **excluding** the following topics:
|
||||
- Llama Chat, the built-in chat app (`ui/apps/chat`)
|
||||
- Web UI
|
||||
- Features marked as experimental
|
||||
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
||||
- Bugs that can lead to Denial-of-Service attack
|
||||
|
||||
@@ -122,8 +122,6 @@ if [ ! -z ${GG_BUILD_VULKAN} ]; then
|
||||
MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION="${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers/SPIRV-HeadersConfig.cmake"
|
||||
if [[ -f "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" || -h "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" ]]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DSPIRV-Headers_DIR=${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers"
|
||||
# SPIRV-Headers walks up three directories and appends /include, yielding /usr/local/lib/include; the extra vulkan/ level likely breaks its prefix calculation.
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN_SPIRV_INCLUDE_DIR=/usr/local/include"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -182,33 +180,23 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
|
||||
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXTRA="-E test-llama-archs|test-llama-context|test-fp8-kv"
|
||||
CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
|
||||
fi
|
||||
|
||||
## helpers
|
||||
|
||||
# download a file unless already cached in $MNT
|
||||
# download a file if it does not exist or if it is outdated
|
||||
function gg_wget {
|
||||
local out=$1
|
||||
local url=$2
|
||||
|
||||
local cwd=`pwd`
|
||||
local name=`basename $url`
|
||||
local tmp=.gg-wget.$$.tmp
|
||||
|
||||
mkdir -p $out
|
||||
cd $out
|
||||
|
||||
# cache first: HF artifacts are static, HF sends no Last-Modified for -N
|
||||
if [ ! -s $name ]; then
|
||||
# publish with mv: concurrent runners never read partial files
|
||||
rm -rf $tmp && mkdir $tmp
|
||||
(cd $tmp && wget -nv $url)
|
||||
if [ -s $tmp/$name ]; then
|
||||
mv -f $tmp/$name $name
|
||||
fi
|
||||
rm -rf $tmp
|
||||
fi
|
||||
# should not re-download if file is the same
|
||||
wget -nv -c -N $url
|
||||
|
||||
cd $cwd
|
||||
}
|
||||
@@ -266,7 +254,7 @@ function gg_run_ctest_release {
|
||||
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
(time ctest -C Release --output-on-failure -L main ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
(time ctest -C Release --output-on-failure -L 'main|python' ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
else
|
||||
(time ctest -C Release --output-on-failure -L main -E test-opt ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
fi
|
||||
@@ -298,6 +286,25 @@ function gg_run_test_llama_archs_tensor_split {
|
||||
set +e
|
||||
}
|
||||
|
||||
# test_llama_archs_models
|
||||
|
||||
function gg_run_test_llama_archs_models {
|
||||
cd ${SRC}
|
||||
|
||||
set -e
|
||||
|
||||
# TODO: fix and re-enable `test-llama-archs` on OpenVINO
|
||||
# TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
|
||||
if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
|
||||
# generate the dummy models used by the model-dependent tests
|
||||
./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
|
||||
fi
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
# test_scripts
|
||||
|
||||
function gg_run_test_scripts {
|
||||
@@ -305,8 +312,8 @@ function gg_run_test_scripts {
|
||||
|
||||
set -e
|
||||
|
||||
(cd ./tools/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$OUT/scratch") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||
(cd ./tools/quantize && time bash tests.sh "$SRC/build-ci-release/bin" "$OUT/scratch") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||
(cd ./tools/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||
(cd ./tools/quantize && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||
|
||||
set +e
|
||||
}
|
||||
@@ -367,14 +374,10 @@ function gg_run_qwen3_0_6b {
|
||||
|
||||
|
||||
gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip
|
||||
|
||||
rm -rf $OUT/scratch/qwen3-0.6B $OUT/scratch/wikitext
|
||||
mkdir -p $OUT/scratch/qwen3-0.6B
|
||||
unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d $OUT/scratch/wikitext/
|
||||
unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
|
||||
|
||||
path_models="../models-mnt/qwen3/0.6B"
|
||||
path_local="$OUT/scratch/qwen3-0.6B"
|
||||
path_wiki="$OUT/scratch/wikitext/wikitext-2-raw"
|
||||
path_wiki="../models-mnt/wikitext/wikitext-2-raw"
|
||||
|
||||
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||
|
||||
@@ -383,15 +386,72 @@ function gg_run_qwen3_0_6b {
|
||||
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf --outtype f16
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-bf16.gguf --outtype bf16
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf --outtype f16
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-bf16.gguf --outtype bf16
|
||||
|
||||
model_f16="${path_local}/ggml-model-f16.gguf"
|
||||
model_bf16="${path_local}/ggml-model-bf16.gguf"
|
||||
model_q4_0="${path_local}/ggml-model-q4_0.gguf"
|
||||
model_f16="${path_models}/ggml-model-f16.gguf"
|
||||
model_bf16="${path_models}/ggml-model-bf16.gguf"
|
||||
model_q8_0="${path_models}/ggml-model-q8_0.gguf"
|
||||
model_q4_0="${path_models}/ggml-model-q4_0.gguf"
|
||||
model_q4_1="${path_models}/ggml-model-q4_1.gguf"
|
||||
model_q5_0="${path_models}/ggml-model-q5_0.gguf"
|
||||
model_q5_1="${path_models}/ggml-model-q5_1.gguf"
|
||||
model_q2_k="${path_models}/ggml-model-q2_k.gguf"
|
||||
model_q3_k="${path_models}/ggml-model-q3_k.gguf"
|
||||
model_q4_k="${path_models}/ggml-model-q4_k.gguf"
|
||||
model_q5_k="${path_models}/ggml-model-q5_k.gguf"
|
||||
model_q6_k="${path_models}/ggml-model-q6_k.gguf"
|
||||
|
||||
wiki_test="${path_wiki}/wiki.test.raw"
|
||||
|
||||
./bin/llama-quantize ${model_bf16} ${model_q8_0} q8_0 $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q4_0} q4_0 $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q4_1} q4_1 $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q5_0} q5_0 $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q5_1} q5_1 $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q2_k} q2_k $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q3_k} q3_k $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q4_k} q4_k $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q5_k} q5_k $(nproc)
|
||||
./bin/llama-quantize ${model_bf16} ${model_q6_k} q6_k $(nproc)
|
||||
|
||||
(time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
|
||||
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_f16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_bf16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q8_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q4_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q4_1} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q5_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q5_1} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q2_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q3_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q4_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q5_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_q6_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
|
||||
|
||||
(time ./bin/llama-perplexity --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
(time ./bin/llama-perplexity --model ${model_bf16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
|
||||
fi
|
||||
(time ./bin/llama-perplexity --model ${model_q8_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||
(time ./bin/llama-perplexity --model ${model_q4_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
|
||||
(time ./bin/llama-perplexity --model ${model_q4_1} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
|
||||
(time ./bin/llama-perplexity --model ${model_q5_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
|
||||
(time ./bin/llama-perplexity --model ${model_q5_1} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
|
||||
(time ./bin/llama-perplexity --model ${model_q2_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
|
||||
(time ./bin/llama-perplexity --model ${model_q3_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
|
||||
(time ./bin/llama-perplexity --model ${model_q4_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
|
||||
(time ./bin/llama-perplexity --model ${model_q5_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
|
||||
(time ./bin/llama-perplexity --model ${model_q6_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
|
||||
|
||||
(time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
|
||||
|
||||
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||
|
||||
function check_ppl {
|
||||
qnt="$1"
|
||||
ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||
@@ -405,52 +465,20 @@ function gg_run_qwen3_0_6b {
|
||||
return 0
|
||||
}
|
||||
|
||||
for qnt in q8_0 q4_0 q4_1 q5_0 q5_1 q2_k q3_k q4_k q5_k q6_k; do
|
||||
model="${path_local}/ggml-model-${qnt}.gguf"
|
||||
|
||||
./bin/llama-quantize ${model_bf16} ${model} ${qnt} $(nproc)
|
||||
|
||||
(time ./bin/llama-completion -no-cnv --model ${model} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-${qnt}.log
|
||||
(time ./bin/llama-perplexity --model ${model} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-${qnt}.log
|
||||
|
||||
# note: ppl > 20.0 for q2_k with this model
|
||||
if [ "${qnt}" != "q2_k" ]; then
|
||||
check_ppl "${qnt}" "$(cat $OUT/${ci}-tg-${qnt}.log | grep "^\[1\]")"
|
||||
fi
|
||||
|
||||
if [ "${qnt}" = "q4_0" ]; then
|
||||
# q4_0 is kept for other tests
|
||||
q4_0="${path_models}/ggml-model-q4_0.gguf"
|
||||
ln ${model} "$q4_0" 2>/dev/null || { cp ${model} $q4_0.tmp.$$ && mv -f $q4_0.tmp.$$ $q4_0; }
|
||||
else
|
||||
rm -f ${model}
|
||||
fi
|
||||
done
|
||||
|
||||
(time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
|
||||
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_f16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||
(time ./bin/llama-completion -no-cnv --model ${model_bf16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
|
||||
|
||||
(time ./bin/llama-perplexity --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
(time ./bin/llama-perplexity --model ${model_bf16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
|
||||
fi
|
||||
|
||||
(time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
|
||||
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
||||
|
||||
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")"
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
|
||||
fi
|
||||
|
||||
# scratch is rebuilt on every run; q4_0 is already published to $MNT
|
||||
rm -rf $OUT/scratch/qwen3-0.6B $OUT/scratch/wikitext
|
||||
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")"
|
||||
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")"
|
||||
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")"
|
||||
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")"
|
||||
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")"
|
||||
#check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model
|
||||
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")"
|
||||
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")"
|
||||
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")"
|
||||
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")"
|
||||
|
||||
set +e
|
||||
}
|
||||
@@ -473,8 +501,6 @@ function gg_run_embd_bge_small {
|
||||
gg_wget models-mnt/bge-small/1_Pooling https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json
|
||||
|
||||
path_models="../models-mnt/bge-small"
|
||||
path_local="$OUT/scratch/bge-small"
|
||||
rm -rf $path_local && mkdir -p $path_local
|
||||
|
||||
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||
|
||||
@@ -483,10 +509,10 @@ function gg_run_embd_bge_small {
|
||||
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf
|
||||
|
||||
model_f16="${path_local}/ggml-model-f16.gguf"
|
||||
model_q8_0="${path_local}/ggml-model-q8_0.gguf"
|
||||
model_f16="${path_models}/ggml-model-f16.gguf"
|
||||
model_q8_0="${path_models}/ggml-model-q8_0.gguf"
|
||||
|
||||
./bin/llama-quantize ${model_f16} ${model_q8_0} q8_0
|
||||
|
||||
@@ -495,8 +521,6 @@ function gg_run_embd_bge_small {
|
||||
(time ./bin/llama-embedding --model ${model_f16} -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||
(time ./bin/llama-embedding --model ${model_q8_0} -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||
|
||||
rm -rf $path_local
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
@@ -513,8 +537,6 @@ function gg_run_rerank_tiny {
|
||||
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/vocab.json
|
||||
|
||||
path_models="../models-mnt/rerank-tiny"
|
||||
path_local="$OUT/scratch/rerank-tiny"
|
||||
rm -rf $path_local && mkdir -p $path_local
|
||||
|
||||
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||
|
||||
@@ -523,9 +545,9 @@ function gg_run_rerank_tiny {
|
||||
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf
|
||||
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf
|
||||
|
||||
model_f16="${path_local}/ggml-model-f16.gguf"
|
||||
model_f16="${path_models}/ggml-model-f16.gguf"
|
||||
|
||||
(time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
|
||||
|
||||
@@ -555,8 +577,6 @@ function gg_run_rerank_tiny {
|
||||
check_score "rerank score 1" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 1")" "0.00" "0.05" | tee -a $OUT/${ci}-rk-f16.log
|
||||
check_score "rerank score 2" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 2")" "0.10" "0.30" | tee -a $OUT/${ci}-rk-f16.log
|
||||
|
||||
rm -rf $path_local
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
@@ -663,11 +683,11 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
ln -sfn ${mnt_models} ${SRC}/models-mnt
|
||||
|
||||
# Create a fresh python3 venv and enter it
|
||||
if ! python3 -m venv "$OUT/venv"; then
|
||||
echo "Error: Failed to create Python virtual environment at $OUT/venv."
|
||||
if ! python3 -m venv "$MNT/venv"; then
|
||||
echo "Error: Failed to create Python virtual environment at $MNT/venv."
|
||||
exit 1
|
||||
fi
|
||||
source "$OUT/venv/bin/activate"
|
||||
source "$MNT/venv/bin/activate"
|
||||
|
||||
pip install -r ${SRC}/requirements.txt --disable-pip-version-check
|
||||
pip install --editable gguf-py --disable-pip-version-check
|
||||
@@ -680,6 +700,7 @@ test $ret -eq 0 && gg_run ctest_release
|
||||
|
||||
test $ret -eq 0 && gg_run test_backend_ops
|
||||
|
||||
test $ret -eq 0 && gg_run test_llama_archs_models
|
||||
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
|
||||
|
||||
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ include("ggml/cmake/common.cmake")
|
||||
|
||||
function(llama_add_compile_flags)
|
||||
if (LLAMA_FATAL_WARNINGS)
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM")
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
list(APPEND C_FLAGS -Werror)
|
||||
list(APPEND CXX_FLAGS -Werror)
|
||||
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
||||
|
||||
@@ -2,11 +2,11 @@ get_filename_component(DEST_DIR "${DEST}" DIRECTORY)
|
||||
file(MAKE_DIRECTORY "${DEST_DIR}")
|
||||
|
||||
if(NOT EXISTS "${DEST}")
|
||||
message(STATUS "Downloading ${NAME} from ${REPO}...")
|
||||
message(STATUS "Downloading ${NAME} from ggml-org/models...")
|
||||
endif()
|
||||
|
||||
file(DOWNLOAD
|
||||
"https://huggingface.co/${REPO}/resolve/main/${NAME}?download=true"
|
||||
"https://huggingface.co/ggml-org/models/resolve/main/${NAME}?download=true"
|
||||
"${DEST}"
|
||||
TLS_VERIFY ON
|
||||
EXPECTED_HASH ${HASH}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
set(LLAMA_WIN_APP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/../common/win-app.manifest")
|
||||
|
||||
if(WIN32)
|
||||
if(MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
|
||||
set(LLAMA_WIN_APP_MSVC_ABI TRUE)
|
||||
else()
|
||||
set(LLAMA_WIN_APP_MSVC_ABI FALSE)
|
||||
endif()
|
||||
|
||||
if(NOT LLAMA_WIN_APP_MSVC_ABI)
|
||||
if(CMAKE_VERSION VERSION_LESS 3.21)
|
||||
message(FATAL_ERROR "CMake 3.21 or newer is required to build for Windows with the GNU ABI")
|
||||
endif()
|
||||
enable_language(RC)
|
||||
set(LLAMA_WIN_APP_MANIFEST_RC "${CMAKE_CURRENT_BINARY_DIR}/win-app-manifest.rc")
|
||||
file(CONFIGURE OUTPUT "${LLAMA_WIN_APP_MANIFEST_RC}" CONTENT "1 24 \"@LLAMA_WIN_APP_MANIFEST@\"\n" @ONLY)
|
||||
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
function(llama_add_app_manifest target)
|
||||
if(NOT WIN32)
|
||||
return()
|
||||
endif()
|
||||
get_target_property(_aliased ${target} ALIASED_TARGET)
|
||||
if(_aliased)
|
||||
return()
|
||||
endif()
|
||||
get_target_property(_type ${target} TYPE)
|
||||
if(NOT _type STREQUAL "EXECUTABLE")
|
||||
return()
|
||||
endif()
|
||||
if(LLAMA_WIN_APP_MSVC_ABI)
|
||||
target_sources(${target} PRIVATE "${LLAMA_WIN_APP_MANIFEST}")
|
||||
else()
|
||||
if(NOT TARGET llama-win-app-manifest-res)
|
||||
add_library(llama-win-app-manifest-res OBJECT ${LLAMA_WIN_APP_MANIFEST_RC})
|
||||
set_source_files_properties(${LLAMA_WIN_APP_MANIFEST_RC} PROPERTIES OBJECT_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
|
||||
endif()
|
||||
target_link_libraries(${target} PRIVATE "$<TARGET_OBJECTS:llama-win-app-manifest-res>")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(llama_apply_app_manifests_dir dir)
|
||||
get_property(_targets DIRECTORY "${dir}" PROPERTY BUILDSYSTEM_TARGETS)
|
||||
foreach(_target IN LISTS _targets)
|
||||
llama_add_app_manifest(${_target})
|
||||
endforeach()
|
||||
get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES)
|
||||
foreach(_subdir IN LISTS _subdirs)
|
||||
llama_apply_app_manifests_dir("${_subdir}")
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
function(llama_apply_app_manifests)
|
||||
if(NOT WIN32)
|
||||
return()
|
||||
endif()
|
||||
get_property(_subdirs DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY SUBDIRECTORIES)
|
||||
foreach(_dir IN LISTS _subdirs)
|
||||
get_filename_component(_name "${_dir}" NAME)
|
||||
if(_name STREQUAL "app" OR _name STREQUAL "examples" OR _name STREQUAL "pocs" OR _name STREQUAL "tests" OR _name STREQUAL "tools")
|
||||
llama_apply_app_manifests_dir("${_dir}")
|
||||
endif()
|
||||
endforeach()
|
||||
endfunction()
|
||||
@@ -66,8 +66,6 @@ add_library(${TARGET}
|
||||
chat-diff-analyzer.cpp
|
||||
chat-peg-parser.cpp
|
||||
chat-peg-parser.h
|
||||
chat-tts-norm.cpp
|
||||
chat-tts-norm.h
|
||||
chat.cpp
|
||||
chat.h
|
||||
common.cpp
|
||||
@@ -80,8 +78,6 @@ add_library(${TARGET}
|
||||
download.h
|
||||
fit.cpp
|
||||
fit.h
|
||||
file-lock.cpp
|
||||
file-lock.h
|
||||
hf-cache.cpp
|
||||
hf-cache.h
|
||||
http.h
|
||||
@@ -188,8 +184,3 @@ if (LLAMA_LLGUIDANCE)
|
||||
endif()
|
||||
|
||||
target_link_libraries(${TARGET} PUBLIC llama Threads::Threads)
|
||||
|
||||
if (CMAKE_SYSTEM_NAME MATCHES "AIX")
|
||||
# AIX's flock() function comes from libbsd.a
|
||||
target_link_libraries(${TARGET} PRIVATE -lbsd)
|
||||
endif()
|
||||
+111
-197
@@ -4,7 +4,6 @@
|
||||
#include "chat.h"
|
||||
#include "common.h"
|
||||
#include "download.h"
|
||||
#include "http.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "json.h"
|
||||
#include "llama.h"
|
||||
@@ -13,6 +12,16 @@
|
||||
#include "speculative.h"
|
||||
#include "preset.h"
|
||||
|
||||
// fix problem with std::min and std::max
|
||||
#if defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#ifndef NOMINMAX
|
||||
# define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#include <shellapi.h>
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
#include <climits>
|
||||
@@ -29,6 +38,20 @@
|
||||
#include <thread> // for hardware_concurrency
|
||||
#include <vector>
|
||||
|
||||
#ifndef __EMSCRIPTEN__
|
||||
#ifdef __linux__
|
||||
#include <linux/limits.h>
|
||||
#elif defined(_WIN32)
|
||||
# if !defined(PATH_MAX)
|
||||
# define PATH_MAX MAX_PATH
|
||||
# endif
|
||||
#elif defined(_AIX)
|
||||
#include <sys/limits.h>
|
||||
#else
|
||||
#include <sys/syslimits.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
|
||||
|
||||
using json = common_json;
|
||||
@@ -282,7 +305,6 @@ const std::vector<ggml_type> kv_cache_types = {
|
||||
GGML_TYPE_F32,
|
||||
GGML_TYPE_F16,
|
||||
GGML_TYPE_BF16,
|
||||
GGML_TYPE_F8_E4M3,
|
||||
GGML_TYPE_Q8_0,
|
||||
GGML_TYPE_Q4_0,
|
||||
GGML_TYPE_Q4_1,
|
||||
@@ -329,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
|
||||
static std::string get_default_local_path(const std::string & url) {
|
||||
auto f = string_split<std::string>(url, '#').front();
|
||||
f = string_split<std::string>(f, '?').front();
|
||||
return fs_get_cache_file(string_split<std::string>(f, '/').back()).string();
|
||||
return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
|
||||
}
|
||||
|
||||
static bool spec_types_is_default(const common_params & params) {
|
||||
@@ -349,14 +371,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_dflash2 = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_lilicorr = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_eagle3 = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();
|
||||
@@ -381,7 +395,7 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
opts.offline = params.offline;
|
||||
opts.download_mtp = spec_type_draft_mtp;
|
||||
opts.download_eagle3 = spec_type_draft_eagle3;
|
||||
opts.download_dflash = spec_type_draft_dflash || spec_type_draft_dflash2 || spec_type_draft_lilicorr;
|
||||
opts.download_dflash = spec_type_draft_dflash;
|
||||
opts.download_dspark = spec_type_draft_dspark;
|
||||
opts.download_mmproj = use_mmproj && !params.no_mmproj
|
||||
&& params.mmproj.path.empty() && params.mmproj.url.empty();
|
||||
@@ -409,26 +423,7 @@ bool common_models_handler_is_preset_repo(const common_models_handler & handler)
|
||||
return !handler.plan.preset.url.empty();
|
||||
}
|
||||
|
||||
static common_download_opts download_opts_for_url(const common_download_opts & opts, const std::string & url) {
|
||||
common_download_opts result = opts;
|
||||
try {
|
||||
const auto hf = common_http_parse_url(common_get_model_endpoint());
|
||||
const auto parts = common_http_parse_url(url);
|
||||
|
||||
if (parts.scheme == hf.scheme &&
|
||||
parts.host == hf.host &&
|
||||
parts.port == hf.port &&
|
||||
string_starts_with(result.bearer_token, "hf_")) {
|
||||
return result;
|
||||
}
|
||||
} catch (const std::exception &) {
|
||||
}
|
||||
result.bearer_token.clear();
|
||||
return result;
|
||||
}
|
||||
|
||||
static std::vector<common_download_task> build_url_tasks(const common_params_model & model, common_download_opts opts) {
|
||||
opts = download_opts_for_url(opts, model.url);
|
||||
auto parts = common_download_get_all_parts(model.url);
|
||||
std::vector<common_download_task> tasks;
|
||||
|
||||
@@ -475,14 +470,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
auto opts = handler.opts; // copy
|
||||
opts.callback = callback;
|
||||
|
||||
// another process may hold the cache lock during finalization:
|
||||
// let the download callback cancel that wait too
|
||||
auto finalize = [callback](const hf_cache::hf_file & f) {
|
||||
return hf_cache::finalize_file(f, [callback]() {
|
||||
return !(callback && callback->is_cancelled());
|
||||
});
|
||||
};
|
||||
|
||||
// handle plain "url" if needed
|
||||
auto handle_url = [&](common_params_model & model) {
|
||||
if (!model.url.empty()) {
|
||||
@@ -517,7 +504,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
common_download_task task;
|
||||
task.url = params.mmproj.url;
|
||||
task.local_path = params.mmproj.path;
|
||||
task.opts = download_opts_for_url(opts, task.url);
|
||||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
}
|
||||
bool had_spec_url = false;
|
||||
@@ -525,13 +512,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
common_download_task task;
|
||||
task.url = params.speculative.draft.mparams.url;
|
||||
task.local_path = params.speculative.draft.mparams.path;
|
||||
task.opts = download_opts_for_url(opts, task.url);
|
||||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
had_spec_url = true;
|
||||
}
|
||||
|
||||
// handle hf_plan tasks
|
||||
auto add_tasks = [&opts, &tasks, &finalize](const hf_cache::hf_files & model_files,
|
||||
auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files,
|
||||
const hf_cache::hf_file & primary,
|
||||
common_params_model & model) {
|
||||
for (size_t i = 0; i < model_files.size(); ++i) {
|
||||
@@ -540,9 +527,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(model_file, opts, [&, is_primary]() {
|
||||
if (is_primary) {
|
||||
// the primary file is the first split (00001-of), use it as model path
|
||||
model.path = finalize(model_file);
|
||||
model.path = hf_cache::finalize_file(model_file);
|
||||
} else {
|
||||
finalize(model_file);
|
||||
hf_cache::finalize_file(model_file);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -594,9 +581,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan_spec.mtp, opts, [&]() {
|
||||
// only use the discovered MTP head when no draft path is set yet
|
||||
if (params.speculative.draft.mparams.path.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan_spec.mtp);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.mtp);
|
||||
} else {
|
||||
finalize(plan_spec.mtp);
|
||||
hf_cache::finalize_file(plan_spec.mtp);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -604,9 +591,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan_spec.dflash, opts, [&]() {
|
||||
// only use the discovered DFlash sidecar when no draft path is set yet
|
||||
if (params.speculative.draft.mparams.path.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan_spec.dflash);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dflash);
|
||||
} else {
|
||||
finalize(plan_spec.dflash);
|
||||
hf_cache::finalize_file(plan_spec.dflash);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -614,9 +601,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan_spec.eagle3, opts, [&]() {
|
||||
// only use the discovered Eagle3 sidecar when no draft path is set yet
|
||||
if (params.speculative.draft.mparams.path.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan_spec.eagle3);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.eagle3);
|
||||
} else {
|
||||
finalize(plan_spec.eagle3);
|
||||
hf_cache::finalize_file(plan_spec.eagle3);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -624,9 +611,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan_spec.dspark, opts, [&]() {
|
||||
// only use the discovered DSpark sidecar when no draft path is set yet
|
||||
if (params.speculative.draft.mparams.path.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan_spec.dspark);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark);
|
||||
} else {
|
||||
finalize(plan_spec.dspark);
|
||||
hf_cache::finalize_file(plan_spec.dspark);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -647,16 +634,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
}
|
||||
if (!plan.mmproj.local_path.empty()) {
|
||||
tasks.emplace_back(plan.mmproj, opts, [&]() {
|
||||
params.mmproj.path = finalize(plan.mmproj);
|
||||
params.mmproj.path = hf_cache::finalize_file(plan.mmproj);
|
||||
});
|
||||
}
|
||||
if (!plan.mtp.local_path.empty() && !had_spec_url) {
|
||||
tasks.emplace_back(plan.mtp, opts, [&]() {
|
||||
// only fall back to the discovered MTP head when no draft was explicitly provided
|
||||
if (params.speculative.draft.mparams.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan.mtp);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.mtp);
|
||||
} else {
|
||||
finalize(plan.mtp);
|
||||
hf_cache::finalize_file(plan.mtp);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -664,9 +651,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan.dflash, opts, [&]() {
|
||||
// only fall back to the discovered DFlash sidecar when no draft was explicitly provided
|
||||
if (params.speculative.draft.mparams.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan.dflash);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dflash);
|
||||
} else {
|
||||
finalize(plan.dflash);
|
||||
hf_cache::finalize_file(plan.dflash);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -674,9 +661,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan.eagle3, opts, [&]() {
|
||||
// only fall back to the discovered Eagle3 sidecar when no draft was explicitly provided
|
||||
if (params.speculative.draft.mparams.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan.eagle3);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.eagle3);
|
||||
} else {
|
||||
finalize(plan.eagle3);
|
||||
hf_cache::finalize_file(plan.eagle3);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -684,9 +671,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan.dspark, opts, [&]() {
|
||||
// only fall back to the discovered DSpark sidecar when no draft was explicitly provided
|
||||
if (params.speculative.draft.mparams.empty()) {
|
||||
params.speculative.draft.mparams.path = finalize(plan.dspark);
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark);
|
||||
} else {
|
||||
finalize(plan.dspark);
|
||||
hf_cache::finalize_file(plan.dspark);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -694,7 +681,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
tasks.emplace_back(plan.preset, opts, [&]() {
|
||||
// if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
|
||||
params.models_preset_hf = params.model.hf_repo; // only for showing a warning
|
||||
params.models_preset = finalize(plan.preset);
|
||||
params.models_preset = hf_cache::finalize_file(plan.preset);
|
||||
// clear the model so the server starts in router mode
|
||||
params.model.path.clear();
|
||||
params.model.hf_repo.clear();
|
||||
@@ -739,7 +726,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
|
||||
std::vector<std::filesystem::path> paths;
|
||||
|
||||
#if defined(_WIN32)
|
||||
const std::filesystem::path program_data = common_get_env("PROGRAMDATA");
|
||||
const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
|
||||
if (!program_data.empty()) {
|
||||
paths.push_back(program_data / "llama.cpp" / "config.ini");
|
||||
}
|
||||
@@ -767,7 +754,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
|
||||
common_preset_context ctx(ex);
|
||||
ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
|
||||
for (const auto & path : found) {
|
||||
LOG_INF("using config file: %s\n", path.string().c_str());
|
||||
LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
|
||||
common_preset global;
|
||||
common_presets presets = ctx.load_from_ini(path, global);
|
||||
global.apply_to_params(params);
|
||||
@@ -912,10 +899,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
|
||||
}
|
||||
|
||||
if (params.sampling.ignore_eos && !common_grammar_value(params.sampling.grammar).empty()) {
|
||||
throw std::invalid_argument("error: --ignore-eos is not supported with --grammar or --json-schema\n");
|
||||
}
|
||||
|
||||
const bool skip_model_download =
|
||||
// server will call common_params_handle_models() later, so we skip it here
|
||||
ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||
|
||||
@@ -933,7 +916,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
// TODO @ngxson : maybe show a list of available models in CLI in this case
|
||||
bool can_skip_model = params.usage || params.completion || !params.server_base.empty();
|
||||
if (!can_skip_model && params.model.path.empty()) {
|
||||
throw std::invalid_argument("error: --model or -hf is required\n");
|
||||
throw std::invalid_argument("error: --model is required\n");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -955,8 +938,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
}
|
||||
|
||||
const bool mcp_enabled = !params.mcp_servers_config.empty() || !params.mcp_servers_json.empty();
|
||||
if ((!params.server_tools.empty() || mcp_enabled || params.ui_mcp_proxy) && !params.cors_origins_explicit) {
|
||||
LOG_WRN("server tools, MCP servers or MCP proxy are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
|
||||
if ((!params.server_tools.empty() || mcp_enabled) && !params.cors_origins_explicit) {
|
||||
LOG_WRN("server tools or MCP servers are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
|
||||
params.cors_origins = "localhost";
|
||||
}
|
||||
|
||||
@@ -1268,7 +1251,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
|
||||
return true;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
struct utf8_argv {
|
||||
std::vector<std::string> buf;
|
||||
std::vector<char*> ptrs;
|
||||
};
|
||||
|
||||
static utf8_argv make_utf8_argv() {
|
||||
utf8_argv out;
|
||||
int wargc = 0;
|
||||
LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
|
||||
if (!wargv) return out;
|
||||
|
||||
out.buf.reserve(wargc);
|
||||
for (int i = 0; i < wargc; ++i) {
|
||||
int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);
|
||||
if (n <= 0) { out.buf.emplace_back(); continue; }
|
||||
auto& s = out.buf.emplace_back();
|
||||
s.resize(static_cast<size_t>(n - 1));
|
||||
(void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);
|
||||
}
|
||||
LocalFree(wargv);
|
||||
|
||||
out.ptrs.reserve(out.buf.size() + 1);
|
||||
for (auto& s : out.buf) out.ptrs.push_back(s.data());
|
||||
out.ptrs.push_back(nullptr);
|
||||
return out;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
|
||||
#ifdef _WIN32
|
||||
auto utf8 = make_utf8_argv();
|
||||
// repair argv only when it matches the process command line
|
||||
if (static_cast<int>(utf8.buf.size()) == argc) {
|
||||
argv = utf8.ptrs.data();
|
||||
}
|
||||
#endif
|
||||
|
||||
auto ctx_arg = common_params_parser_init(params, ex, print_usage);
|
||||
const common_params params_org = ctx_arg.params; // the example can modify the default params
|
||||
|
||||
@@ -1459,7 +1479,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
));
|
||||
add_opt(common_arg(
|
||||
{"--server-base"}, "URL",
|
||||
string_format("connect to this server instead of starting a new one, example: 'http://localhost:9931' (default: none)"),
|
||||
string_format("connect to this server instead of starting a new one, example: 'http://localhost:8080' (default: none)"),
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.server_base = value;
|
||||
}
|
||||
@@ -1697,30 +1717,12 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"-cram", "--cache-ram"}, "N",
|
||||
string_format("set the maximum cache size in MiB (default: %d, 0 for decision models, -1 - no limit, 0 - disable)"
|
||||
string_format("set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)"
|
||||
"[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)", params.cache_ram_mib),
|
||||
[](common_params & params, int value) {
|
||||
params.cache_ram_mib = value;
|
||||
params.cache_ram_explicit = true;
|
||||
}
|
||||
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
|
||||
add_opt(common_arg(
|
||||
{"--cache-disk-path"}, "PATH",
|
||||
"save the state of each slot with a prompt to this directory on shutdown, and restore it for a prompt that starts with it after a restart (default: disabled)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.cache_disk_path = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_CACHE_DISK_PATH").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--cache-disk-size"}, "N",
|
||||
string_format("size limit of the --cache-disk-path directory in MiB, the least recently used files are removed (default: %d, 0 - no limit)", params.cache_disk_mib),
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
throw std::invalid_argument("cache-disk-size must be non-negative");
|
||||
}
|
||||
params.cache_disk_mib = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_CACHE_DISK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"-kvu", "--kv-unified"},
|
||||
{"-no-kvu", "--no-kv-unified"},
|
||||
@@ -1728,7 +1730,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
[](common_params & params, bool value) {
|
||||
params.kv_unified = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL, LLAMA_EXAMPLE_SPECULATIVE}));
|
||||
).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL}));
|
||||
add_opt(common_arg(
|
||||
{"--cache-idle-slots"},
|
||||
{"--no-cache-idle-slots"},
|
||||
@@ -2530,6 +2532,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.ppl_output_type = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
|
||||
add_opt(common_arg(
|
||||
{"-dt", "--defrag-thold"}, "N",
|
||||
string_format("KV cache defragmentation threshold (DEPRECATED)"),
|
||||
[](common_params & params, const std::string & value) {
|
||||
GGML_UNUSED(params);
|
||||
GGML_UNUSED(value);
|
||||
LOG_WRN("DEPRECATED: --defrag-thold is deprecated and no longer necessary to specify\n");
|
||||
}
|
||||
).set_env("LLAMA_ARG_DEFRAG_THOLD"));
|
||||
if (ex == LLAMA_EXAMPLE_SERVER) {
|
||||
// this is to make sure this option appears in the server-specific section of the help message
|
||||
add_opt(common_arg(
|
||||
@@ -2775,17 +2786,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.moe_cache_size = (size_t) value*1024*1024;
|
||||
}
|
||||
).set_env("LLAMA_ARG_MOE_CACHE_MIB"));
|
||||
add_opt(common_arg(
|
||||
{"--moe-cache-cpu-share"}, "F",
|
||||
string_format("share of the MoE cache misses that the CPU computes from the host experts instead of uploading them to the GPU once the cache is full, from 0 to 1 (default: %.2f)", (double) params.moe_cache_cpu_share),
|
||||
[](common_params & params, const std::string & value) {
|
||||
const float share = std::stof(value);
|
||||
if (share < 0.0f || share > 1.0f) {
|
||||
throw std::invalid_argument("invalid value");
|
||||
}
|
||||
params.moe_cache_cpu_share = share;
|
||||
}
|
||||
).set_env("LLAMA_ARG_MOE_CACHE_CPU_SHARE"));
|
||||
add_opt(common_arg(
|
||||
{"-ncffn", "--n-cpu-ffn"}, "N",
|
||||
"keep the dense FFN weights of the first N layers in the CPU\n"
|
||||
@@ -2850,7 +2850,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -2916,7 +2916,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -3053,16 +3053,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.model.path = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
|
||||
add_opt(common_arg(
|
||||
{"--ple", "--shard"}, "FNAME,...",
|
||||
"extra GGUF shards for the model, comma separated; may be given more than once\n"
|
||||
"paths may live anywhere, including on different drives; --ple is an alias for --shard",
|
||||
[](common_params & params, const std::string & value) {
|
||||
for (const auto & item : parse_csv_row(value)) {
|
||||
params.model.shards.push_back(item);
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD"));
|
||||
add_opt(common_arg(
|
||||
{"-mu", "--model-url"}, "MODEL_URL",
|
||||
"model download url (default: unused)",
|
||||
@@ -3381,7 +3371,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
{"--cors-origins"}, "ORIGINS",
|
||||
string_format(
|
||||
"comma-separated list of allowed origins for CORS (default: %s)\n"
|
||||
"if set to special value 'localhost', only allow same-origin requests (e.g. the built-in UI)",
|
||||
"if set to special value 'localhost', reflect the Origin header only if it is localhost",
|
||||
params.cors_origins.c_str()),
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.cors_origins = value;
|
||||
@@ -3437,20 +3427,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused unless --ui-mcp-proxy-allow-private-urls is set - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
"experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY"));
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy-allow-private-urls"},
|
||||
"allow the MCP CORS proxy to reach local or private addresses (e.g. MCP servers on localhost or LAN) - anyone who can reach the proxy can then reach your local network (default: disabled, only public addresses)\n"
|
||||
"note: only has effect with --ui-mcp-proxy",
|
||||
[](common_params & params) {
|
||||
params.ui_mcp_proxy_allow_private_urls = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY_ALLOW_PRIVATE_URLS"));
|
||||
add_opt(common_arg(
|
||||
{"--tools"}, "TOOL1,TOOL2,...",
|
||||
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
|
||||
@@ -3472,13 +3453,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.server_tools_runtime = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME"));
|
||||
add_opt(common_arg(
|
||||
{"--allow-unauthenticated-tools"},
|
||||
"allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)",
|
||||
[](common_params & params) {
|
||||
params.server_tools_allow_unauth = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS"));
|
||||
add_opt(common_arg(
|
||||
{"--mcp-servers-config"}, "PATH",
|
||||
"experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n"
|
||||
@@ -3514,31 +3488,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui", "--webui"},
|
||||
{"--no-ui", "--no-webui"},
|
||||
string_format("whether to enable Llama Chat, the built-in chat app (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
[](common_params & params, bool value) {
|
||||
params.ui = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI"));
|
||||
add_opt(common_arg(
|
||||
{"--connect"},
|
||||
string_format("(experimental) open a peer-to-peer tunnel for server using llama-connect (default: %s)", params.server_connect ? "enabled" : "disabled"),
|
||||
[](common_params & params) {
|
||||
params.server_connect = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CONNECT"));
|
||||
add_opt(common_arg(
|
||||
{"--connect-code"}, "CODE",
|
||||
"manually specify a 40-character code for --connect (default: generate a new one for each run)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
std::string val = value;
|
||||
string_replace_all(val, "-", "");
|
||||
string_replace_all(val, " ", "");
|
||||
if (val.size() != 40) {
|
||||
throw std::invalid_argument(string_format("error: invalid connect code '%s', must be 40 characters\n", value.c_str()));
|
||||
}
|
||||
params.server_connect_code = val;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CONNECT_CODE"));
|
||||
add_opt(common_arg(
|
||||
{"--embedding", "--embeddings"},
|
||||
string_format("restrict to only support embedding use case; use only with dedicated embedding models (default: %s)", params.embedding ? "enabled" : "disabled"),
|
||||
@@ -3689,7 +3643,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.slot_save_path += DIRECTORY_SEPARATOR;
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SLOT_SAVE_PATH"));
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--media-path"}, "PATH",
|
||||
"directory for loading local media files; files can be accessed via file:// URLs using relative paths (default: disabled)",
|
||||
@@ -3704,13 +3658,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--media-allow-private-urls"},
|
||||
"allow http(s) media URLs (e.g. image_url) that point to local or private addresses (default: disabled, only public addresses)",
|
||||
[](common_params & params) {
|
||||
params.media_allow_private_urls = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MEDIA_ALLOW_PRIVATE_URLS"));
|
||||
add_opt(common_arg(
|
||||
{"--models-dir"}, "PATH",
|
||||
"directory containing models for the router server (default: disabled)",
|
||||
@@ -3732,13 +3679,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.models_max = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MAX"));
|
||||
add_opt(common_arg(
|
||||
{"--models-memory-margin"}, "N",
|
||||
string_format("for router server, MiB of device memory to keep free per device, unload LRU models until a new one fits (default: %d, 0 = disabled)", params.models_memory_margin),
|
||||
[](common_params & params, int value) {
|
||||
params.models_memory_margin = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MEMORY_MARGIN"));
|
||||
add_opt(common_arg(
|
||||
{"--models-autoload"},
|
||||
{"--no-models-autoload"},
|
||||
@@ -4233,16 +4173,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.speculative.draft.n_min = value;
|
||||
}
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
|
||||
add_opt(common_arg(
|
||||
{"--spec-draft-n-ubatch"}, "N",
|
||||
string_format("physical maximum batch size of the draft context, use a smaller value to spare VRAM (default: %d - same as --n-ubatch)", params.speculative.draft.n_ubatch),
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
throw std::invalid_argument("invalid value");
|
||||
}
|
||||
params.speculative.draft.n_ubatch = value;
|
||||
}
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_UBATCH"));
|
||||
add_opt(common_arg(
|
||||
{"--spec-synth-len"}, "L",
|
||||
"target mean synthetic acceptance length, including the target token (benchmarking only)",
|
||||
@@ -4531,19 +4461,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--tts-lang"}, "FNAME",
|
||||
"language (ISO 639-1) for audio generation\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
"see tts/README.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_lang = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-voice"}, "VOICE",
|
||||
"voice for audio generation (model-dependent)\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_voice = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-file"}, "FNAME",
|
||||
"speaker file path for audio generation",
|
||||
@@ -4551,14 +4473,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.tts_speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-text"}, "TEXT",
|
||||
"transcript of --tts-speaker-file, for models that need it to clone the voice\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_text = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
//
|
||||
// diffusion params
|
||||
|
||||
@@ -58,10 +58,11 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
data.parser = autoparser.build_parser(inputs, parser_generation_prompt);
|
||||
auto parser = autoparser.build_parser(inputs, parser_generation_prompt);
|
||||
data.parser = parser.save();
|
||||
|
||||
// Build grammar if tools are present
|
||||
bool has_tools =
|
||||
@@ -77,7 +78,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
// Set grammar triggers based on tool section markers (fall back to per-call markers)
|
||||
|
||||
@@ -322,7 +322,7 @@ std::string apply_template(const common_chat_template & tmpl, const template_par
|
||||
tmpl_params.extra_context["enable_thinking"] = params.enable_thinking;
|
||||
|
||||
try {
|
||||
return common_chat_template_direct_apply(tmpl, tmpl_params).str();
|
||||
return common_chat_template_direct_apply(tmpl, tmpl_params);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_DBG("Template application failed: %s\n", e.what());
|
||||
return ERR_TMPL;
|
||||
|
||||
+49
-67
@@ -131,12 +131,6 @@ static std::string normalize_quotes_to_json(const std::string & input) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Models can emit raw control characters inside string literals; escape them to keep the JSON valid.
|
||||
if ((in_single_quoted || in_double_quoted) && static_cast<unsigned char>(c) < 0x20) {
|
||||
result += escape_json_string_inner(std::string(1, c));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle quote characters
|
||||
if (c == '"') {
|
||||
if (in_single_quoted) {
|
||||
@@ -354,8 +348,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
|
||||
}
|
||||
|
||||
if (is_tool_id && current_tool) {
|
||||
auto sanitized = node.sanitized_text();
|
||||
auto text = trim_trailing_space(sanitized);
|
||||
auto text = trim_trailing_space(node.text);
|
||||
if (text.size() >= 2 && text.front() == '"' && text.back() == '"') {
|
||||
text = text.substr(1, text.size() - 2);
|
||||
}
|
||||
@@ -408,7 +401,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
|
||||
}
|
||||
|
||||
if ((is_arg_value || is_arg_string_value) && current_tool) {
|
||||
std::string value_content = node.sanitized_text();
|
||||
std::string value_content = std::string(node.text);
|
||||
|
||||
std::string value_to_add;
|
||||
if (value_content.empty() && is_arg_string_value) {
|
||||
@@ -562,8 +555,7 @@ common_peg_parser common_chat_peg_builder::python_or_json_value() {
|
||||
common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
const ordered_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared) {
|
||||
bool allow_json_literals) {
|
||||
if (!tools.is_array() || tools.empty()) {
|
||||
return eps();
|
||||
}
|
||||
@@ -621,18 +613,6 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
|
||||
}
|
||||
|
||||
// last, so declared tools win when parsing
|
||||
if (allow_undeclared) {
|
||||
auto ident = rule("tool-undeclared-ident", chars("[a-zA-Z_]", 1, 1) + chars("[a-zA-Z0-9_.]", 0, -1));
|
||||
auto value = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());
|
||||
auto arg = tool_arg(tool_arg_open(tool_arg_name(ident) + literal("=")) + value + tool_arg_close(eps()));
|
||||
auto args = optional(arg + zero_or_more("," + space() + arg));
|
||||
|
||||
tool_choices |= rule("tool-undeclared", tool(tool_open(tool_name(ident) + literal("(")) +
|
||||
space() + tool_args(args) + space() + tool_close(literal(")"))
|
||||
));
|
||||
}
|
||||
|
||||
if (parallel_tool_calls) {
|
||||
return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]";
|
||||
}
|
||||
@@ -986,21 +966,6 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena,
|
||||
}
|
||||
}
|
||||
|
||||
using gemma4 = common_chat_peg_gemma4_mapper;
|
||||
|
||||
// Collects the nearest descendants carrying tag, looking through the rules the parser
|
||||
// wraps around the tagged nodes.
|
||||
static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector<common_peg_ast_id> & out) {
|
||||
for (auto child_id : node.children) {
|
||||
const auto & child = arena.get(child_id);
|
||||
if (child.tag == tag) {
|
||||
out.push_back(child_id);
|
||||
} else {
|
||||
gemma4_collect(arena, child, tag, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {
|
||||
const auto & node = arena.get(id);
|
||||
|
||||
@@ -1008,45 +973,51 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
|
||||
return "";
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) {
|
||||
if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {
|
||||
return std::string(node.text);
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::STRING_CONTENT) {
|
||||
return escape_json_string_inner(node.sanitized_text());
|
||||
if (node.rule == "gemma4-string-content") {
|
||||
return escape_json_string_inner(std::string(node.text));
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::STRING) {
|
||||
if (node.rule == "gemma4-string") {
|
||||
std::string result = "\"";
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
if (!node.is_partial) {
|
||||
result += "\"";
|
||||
if (!node.children.empty()) {
|
||||
result += gemma4_to_json(arena, node.children[0]);
|
||||
if (!node.is_partial) {
|
||||
result += "\"";
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::ARRAY) {
|
||||
if (node.rule == "gemma4-array") {
|
||||
std::string result = "[";
|
||||
for (size_t i = 0; i < node.children.size(); i++) {
|
||||
if (i > 0) {
|
||||
|
||||
bool add_comma = false;
|
||||
for (auto child_id : node.children) {
|
||||
if (add_comma) {
|
||||
result += ',';
|
||||
}
|
||||
result += gemma4_to_json(arena, node.children[i]);
|
||||
add_comma = true;
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
|
||||
if (!node.is_partial) {
|
||||
result += ']';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::DICT_KEY) {
|
||||
auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME);
|
||||
if (node.rule == "gemma4-dict-key-name") {
|
||||
return std::string(node.text);
|
||||
}
|
||||
|
||||
if (node.rule == "gemma4-dict-key") {
|
||||
std::string result = "\"";
|
||||
if (name_id != COMMON_PEG_INVALID_AST_ID) {
|
||||
result += escape_json_string_inner(std::string(arena.get(name_id).text));
|
||||
if (!node.children.empty()) {
|
||||
result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));
|
||||
}
|
||||
if (!node.is_partial) {
|
||||
result += "\":";
|
||||
@@ -1054,29 +1025,40 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.tag == gemma4::DICT) {
|
||||
std::vector<common_peg_ast_id> members;
|
||||
gemma4_collect(arena, node, gemma4::DICT_KV, members);
|
||||
if (node.rule == "gemma4-dict-kv") {
|
||||
std::string result;
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (node.rule == "gemma4-dict") {
|
||||
std::string result = "{";
|
||||
for (size_t i = 0; i < members.size(); i++) {
|
||||
if (i > 0) {
|
||||
|
||||
bool add_comma = false;
|
||||
for (auto child_id : node.children) {
|
||||
if (add_comma) {
|
||||
result += ',';
|
||||
}
|
||||
result += gemma4_to_json(arena, members[i]);
|
||||
add_comma = true;
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
}
|
||||
|
||||
if (!node.is_partial) {
|
||||
result += '}';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// members, rules and other wrappers
|
||||
std::string result;
|
||||
for (auto child_id : node.children) {
|
||||
result += gemma4_to_json(arena, child_id);
|
||||
if (node.rule == "gemma4-value") {
|
||||
if (!node.children.empty()) {
|
||||
return gemma4_to_json(arena, node.children[0]);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
return result;
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {
|
||||
@@ -1213,7 +1195,7 @@ static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena,
|
||||
}
|
||||
|
||||
if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) {
|
||||
return "\"" + escape_json_string_inner(node.sanitized_text()) + (closed ? "\"" : "");
|
||||
return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : "");
|
||||
}
|
||||
|
||||
// Numbers and booleans are written verbatim by the template
|
||||
|
||||
@@ -34,18 +34,6 @@ class common_chat_peg_mapper {
|
||||
|
||||
class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {
|
||||
public:
|
||||
// Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules.
|
||||
static constexpr const char * STRING = "gemma4-string";
|
||||
static constexpr const char * STRING_CONTENT = "gemma4-string-content";
|
||||
static constexpr const char * NUMBER = "gemma4-number";
|
||||
static constexpr const char * BOOL = "gemma4-bool";
|
||||
static constexpr const char * NULL_ = "gemma4-null";
|
||||
static constexpr const char * ARRAY = "gemma4-array";
|
||||
static constexpr const char * DICT = "gemma4-dict";
|
||||
static constexpr const char * DICT_KV = "gemma4-dict-kv";
|
||||
static constexpr const char * DICT_KEY = "gemma4-dict-key";
|
||||
static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name";
|
||||
|
||||
common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}
|
||||
virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);
|
||||
private:
|
||||
@@ -161,11 +149,9 @@ class common_chat_peg_builder : public common_peg_parser_builder {
|
||||
|
||||
// Helper for Python-style function call format: name(arg1="value1", arg2=123)
|
||||
// Used by LFM2 and similar templates
|
||||
// allow_undeclared: also accept calls to names not in tools (any identifier, any arguments)
|
||||
common_peg_parser python_style_tool_calls(const common_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared = false);
|
||||
bool allow_json_literals);
|
||||
|
||||
private:
|
||||
// Python values plus JSON true/false/null.
|
||||
|
||||
@@ -1,902 +0,0 @@
|
||||
#include "chat-tts-norm.h"
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification
|
||||
// keep this file in sync with the python reference: scripts/chat_tts_norm.py
|
||||
|
||||
using namespace jinja;
|
||||
|
||||
//
|
||||
// character classes
|
||||
// they are defined with explicit codepoint ranges, independent of any unicode database version,
|
||||
// so that the C++ and python implementations agree on every codepoint
|
||||
//
|
||||
|
||||
struct tts_norm_range {
|
||||
uint32_t lo;
|
||||
uint32_t hi;
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_SPACE[] = {
|
||||
{0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},
|
||||
{0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_PUNCT[] = {
|
||||
{0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},
|
||||
{0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},
|
||||
{0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},
|
||||
{0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},
|
||||
{0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},
|
||||
{0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},
|
||||
{0x1F000, 0x1FAFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_CONTROL[] = {
|
||||
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},
|
||||
};
|
||||
|
||||
static const tts_norm_range TTS_NORM_HAN[] = {
|
||||
{0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},
|
||||
{0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},
|
||||
};
|
||||
|
||||
template<size_t N>
|
||||
static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {
|
||||
for (const auto & r : ranges) {
|
||||
if (c >= r.lo && c <= r.hi) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_alpha(uint32_t c) {
|
||||
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
|
||||
return true;
|
||||
}
|
||||
if (c == 0xAA || c == 0xB5 || c == 0xBA) {
|
||||
return true;
|
||||
}
|
||||
return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
|
||||
// kind is one of "dspawulhc"
|
||||
static bool tts_norm_class_match(char kind, uint32_t c) {
|
||||
switch (kind) {
|
||||
case 'd': return c >= '0' && c <= '9';
|
||||
case 's': return tts_norm_in(TTS_NORM_SPACE, c);
|
||||
case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);
|
||||
case 'a': return tts_norm_is_alpha(c);
|
||||
case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');
|
||||
case 'u': return c >= 'A' && c <= 'Z';
|
||||
case 'l': return c >= 'a' && c <= 'z';
|
||||
case 'h': return tts_norm_in(TTS_NORM_HAN, c);
|
||||
case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_class_letter(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') {
|
||||
c += 'a' - 'A';
|
||||
}
|
||||
return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_lower(uint32_t c) {
|
||||
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
|
||||
}
|
||||
|
||||
static uint32_t tts_norm_ascii_swapcase(uint32_t c) {
|
||||
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
|
||||
if (c >= 'a' && c <= 'z') return c - ('a' - 'A');
|
||||
return c;
|
||||
}
|
||||
|
||||
static bool tts_norm_is_ascii_alnum(uint32_t c) {
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
|
||||
// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size
|
||||
static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {
|
||||
cpts.clear();
|
||||
offs.clear();
|
||||
size_t i = 0;
|
||||
const size_t n = s.size();
|
||||
while (i < n) {
|
||||
offs.push_back(i);
|
||||
const unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
uint32_t cp = 0;
|
||||
size_t len = 0;
|
||||
if (c < 0x80) {
|
||||
cp = c; len = 1;
|
||||
} else if ((c & 0xE0) == 0xC0) {
|
||||
cp = c & 0x1F; len = 2;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
cp = c & 0x0F; len = 3;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
cp = c & 0x07; len = 4;
|
||||
}
|
||||
bool ok = len > 0 && i + len <= n;
|
||||
for (size_t k = 1; ok && k < len; k++) {
|
||||
const unsigned char cc = static_cast<unsigned char>(s[i + k]);
|
||||
if ((cc & 0xC0) != 0x80) {
|
||||
ok = false;
|
||||
} else {
|
||||
cp = (cp << 6) | (cc & 0x3F);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
cpts.push_back(0xFFFD);
|
||||
i += 1;
|
||||
} else {
|
||||
cpts.push_back(cp);
|
||||
i += len;
|
||||
}
|
||||
}
|
||||
offs.push_back(n);
|
||||
}
|
||||
|
||||
//
|
||||
// AST
|
||||
//
|
||||
|
||||
enum class tts_norm_op {
|
||||
LITERAL, // one codepoint
|
||||
SET, // one codepoint from a set; class escapes like %d are sets with a single class
|
||||
ANY, // any codepoint
|
||||
START, // ^
|
||||
END, // $
|
||||
BOUNDARY, // %b
|
||||
GROUP, // (...), (?:...), (?<name>...)
|
||||
LOOKAHEAD, // (?=...), (?!...)
|
||||
LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint
|
||||
};
|
||||
|
||||
struct tts_norm_class {
|
||||
char kind;
|
||||
bool neg;
|
||||
};
|
||||
|
||||
struct tts_norm_set {
|
||||
bool neg = false;
|
||||
std::vector<std::pair<uint32_t, uint32_t>> ranges;
|
||||
std::vector<tts_norm_class> classes;
|
||||
};
|
||||
|
||||
struct tts_norm_item;
|
||||
using tts_norm_seq = std::vector<tts_norm_item>;
|
||||
|
||||
struct tts_norm_pattern_node {
|
||||
tts_norm_op op = tts_norm_op::GROUP;
|
||||
uint32_t cp = 0; // LITERAL
|
||||
tts_norm_set set; // SET
|
||||
std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item
|
||||
int capture = -1; // GROUP: capture index, -1 for non-capturing
|
||||
bool neg = false; // LOOKAHEAD, LOOKBEHIND
|
||||
};
|
||||
|
||||
static constexpr int TTS_NORM_INF = INT_MAX;
|
||||
static constexpr int TTS_NORM_MAX_REPEAT = 1000;
|
||||
static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;
|
||||
|
||||
struct tts_norm_item {
|
||||
tts_norm_pattern_node atom;
|
||||
int min = 1;
|
||||
int max = 1;
|
||||
};
|
||||
|
||||
static bool tts_norm_is_zero_width(tts_norm_op op) {
|
||||
return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY
|
||||
|| op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;
|
||||
}
|
||||
|
||||
//
|
||||
// parser
|
||||
//
|
||||
|
||||
struct tts_norm_parser {
|
||||
std::vector<uint32_t> s;
|
||||
size_t p = 0;
|
||||
size_t & n_caps;
|
||||
|
||||
tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}
|
||||
|
||||
[[noreturn]] void fail(const std::string & msg) const {
|
||||
throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));
|
||||
}
|
||||
|
||||
bool eof() const { return p >= s.size(); }
|
||||
uint32_t peek() const { return s[p]; }
|
||||
bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }
|
||||
|
||||
void expect(uint32_t c, const char * what) {
|
||||
if (!peek_is(c)) {
|
||||
fail(std::string("expected ") + what);
|
||||
}
|
||||
p++;
|
||||
}
|
||||
|
||||
std::vector<tts_norm_seq> parse_alts() {
|
||||
std::vector<tts_norm_seq> alts;
|
||||
alts.push_back(parse_seq());
|
||||
while (peek_is('|')) {
|
||||
p++;
|
||||
alts.push_back(parse_seq());
|
||||
}
|
||||
return alts;
|
||||
}
|
||||
|
||||
tts_norm_seq parse_seq() {
|
||||
tts_norm_seq seq;
|
||||
while (!eof() && peek() != '|' && peek() != ')') {
|
||||
tts_norm_item item;
|
||||
item.atom = parse_atom();
|
||||
parse_quantifier(item);
|
||||
seq.push_back(std::move(item));
|
||||
}
|
||||
return seq;
|
||||
}
|
||||
|
||||
int parse_int() {
|
||||
if (eof() || peek() < '0' || peek() > '9') {
|
||||
fail("expected number in quantifier");
|
||||
}
|
||||
int v = 0;
|
||||
while (!eof() && peek() >= '0' && peek() <= '9') {
|
||||
v = v * 10 + static_cast<int>(peek() - '0');
|
||||
if (v > TTS_NORM_MAX_REPEAT) {
|
||||
fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));
|
||||
}
|
||||
p++;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
void parse_quantifier(tts_norm_item & item) {
|
||||
if (eof()) {
|
||||
return;
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c != '?' && c != '*' && c != '+' && c != '{') {
|
||||
return;
|
||||
}
|
||||
if (tts_norm_is_zero_width(item.atom.op)) {
|
||||
fail("zero-width assertion cannot be repeated");
|
||||
}
|
||||
p++;
|
||||
if (c == '?') {
|
||||
item.min = 0; item.max = 1;
|
||||
} else if (c == '*') {
|
||||
item.min = 0; item.max = TTS_NORM_INF;
|
||||
} else if (c == '+') {
|
||||
item.min = 1; item.max = TTS_NORM_INF;
|
||||
} else {
|
||||
item.min = parse_int();
|
||||
item.max = item.min;
|
||||
if (peek_is(',')) {
|
||||
p++;
|
||||
item.max = peek_is('}') ? TTS_NORM_INF : parse_int();
|
||||
}
|
||||
expect('}', "'}' to close quantifier");
|
||||
if (item.max < item.min) {
|
||||
fail("quantifier max is smaller than min");
|
||||
}
|
||||
}
|
||||
if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {
|
||||
fail("multiple quantifiers");
|
||||
}
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_group() {
|
||||
// '(' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::GROUP;
|
||||
if (peek_is('?')) {
|
||||
p++;
|
||||
if (eof()) {
|
||||
fail("unterminated group");
|
||||
}
|
||||
const uint32_t c = peek();
|
||||
if (c == ':') {
|
||||
p++;
|
||||
} else if (c == '=' || c == '!') {
|
||||
p++;
|
||||
node.op = tts_norm_op::LOOKAHEAD;
|
||||
node.neg = c == '!';
|
||||
} else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {
|
||||
node.op = tts_norm_op::LOOKBEHIND;
|
||||
node.neg = s[p + 1] == '!';
|
||||
p += 2;
|
||||
tts_norm_item item;
|
||||
if (eof() || peek() == ')') {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
item.atom = parse_atom();
|
||||
if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
if (!peek_is(')')) {
|
||||
fail("lookbehind must contain exactly one codepoint matcher");
|
||||
}
|
||||
p++;
|
||||
node.alts.push_back(tts_norm_seq{});
|
||||
node.alts[0].push_back(std::move(item));
|
||||
return node;
|
||||
} else if (c == 'i') {
|
||||
fail("(?i) is only allowed at the start of the pattern");
|
||||
} else {
|
||||
fail("unknown group type");
|
||||
}
|
||||
} else {
|
||||
node.capture = static_cast<int>(n_caps++);
|
||||
}
|
||||
node.alts = parse_alts();
|
||||
expect(')', "')' to close group");
|
||||
return node;
|
||||
}
|
||||
|
||||
// after '%' inside or outside of a set
|
||||
// returns true if it is a class (stored in cls), false if it is a literal (stored in lit)
|
||||
bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {
|
||||
boundary = false;
|
||||
if (eof()) {
|
||||
fail("dangling '%'");
|
||||
}
|
||||
const uint32_t e = s[p++];
|
||||
if (tts_norm_is_class_letter(e)) {
|
||||
cls.kind = static_cast<char>(tts_norm_ascii_lower(e));
|
||||
cls.neg = e >= 'A' && e <= 'Z';
|
||||
return true;
|
||||
}
|
||||
if (e == 'b' && !in_set) {
|
||||
boundary = true;
|
||||
return false;
|
||||
}
|
||||
if (tts_norm_is_ascii_alnum(e)) {
|
||||
p--;
|
||||
fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");
|
||||
}
|
||||
lit = e;
|
||||
return false;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_set() {
|
||||
// '[' already consumed
|
||||
tts_norm_pattern_node node;
|
||||
node.op = tts_norm_op::SET;
|
||||
if (peek_is('^')) {
|
||||
node.set.neg = true;
|
||||
p++;
|
||||
}
|
||||
bool empty = true;
|
||||
while (true) {
|
||||
if (eof()) {
|
||||
fail("unterminated set");
|
||||
}
|
||||
if (peek() == ']') {
|
||||
p++;
|
||||
break;
|
||||
}
|
||||
empty = false;
|
||||
uint32_t lo = 0;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, lo, boundary)) {
|
||||
node.set.classes.push_back(cls);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
lo = s[p++];
|
||||
}
|
||||
uint32_t hi = lo;
|
||||
if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {
|
||||
p++;
|
||||
if (peek() == '%') {
|
||||
p++;
|
||||
tts_norm_class cls{};
|
||||
bool boundary = false;
|
||||
if (parse_escape(true, cls, hi, boundary)) {
|
||||
fail("character class cannot be used in a range");
|
||||
}
|
||||
} else {
|
||||
hi = s[p++];
|
||||
}
|
||||
if (hi < lo) {
|
||||
fail("invalid range in set");
|
||||
}
|
||||
}
|
||||
node.set.ranges.emplace_back(lo, hi);
|
||||
}
|
||||
if (empty) {
|
||||
fail("empty set");
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
tts_norm_pattern_node parse_atom() {
|
||||
const uint32_t c = s[p++];
|
||||
tts_norm_pattern_node node;
|
||||
switch (c) {
|
||||
case '(':
|
||||
return parse_group();
|
||||
case '[':
|
||||
return parse_set();
|
||||
case '.':
|
||||
node.op = tts_norm_op::ANY;
|
||||
return node;
|
||||
case '^':
|
||||
node.op = tts_norm_op::START;
|
||||
return node;
|
||||
case '$':
|
||||
node.op = tts_norm_op::END;
|
||||
return node;
|
||||
case '%': {
|
||||
tts_norm_class cls{};
|
||||
uint32_t lit = 0;
|
||||
bool boundary = false;
|
||||
if (parse_escape(false, cls, lit, boundary)) {
|
||||
node.op = tts_norm_op::SET;
|
||||
node.set.classes.push_back(cls);
|
||||
} else if (boundary) {
|
||||
node.op = tts_norm_op::BOUNDARY;
|
||||
} else {
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = lit;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
case '?':
|
||||
case '*':
|
||||
case '+':
|
||||
case '{':
|
||||
p--;
|
||||
fail("nothing to repeat");
|
||||
case ']':
|
||||
case '}':
|
||||
p--;
|
||||
fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");
|
||||
default:
|
||||
node.op = tts_norm_op::LITERAL;
|
||||
node.cp = c;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// matcher
|
||||
// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive
|
||||
// (they never give back what they consumed), so there is no backtracking
|
||||
//
|
||||
|
||||
struct tts_norm_matcher {
|
||||
const std::vector<uint32_t> & t;
|
||||
bool icase;
|
||||
std::vector<std::pair<int64_t, int64_t>> caps;
|
||||
|
||||
bool is_w(size_t i) const {
|
||||
return i < t.size() && tts_norm_class_match('w', t[i]);
|
||||
}
|
||||
|
||||
bool set_match(const tts_norm_set & st, uint32_t c) const {
|
||||
auto test = [&](uint32_t x) {
|
||||
for (const auto & r : st.ranges) {
|
||||
if (x >= r.first && x <= r.second) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (const auto & k : st.classes) {
|
||||
if (tts_norm_class_match(k.kind, x) != k.neg) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
bool m = test(c);
|
||||
if (!m && icase) {
|
||||
const uint32_t o = tts_norm_ascii_swapcase(c);
|
||||
if (o != c) {
|
||||
m = test(o);
|
||||
}
|
||||
}
|
||||
return m != st.neg;
|
||||
}
|
||||
|
||||
bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {
|
||||
for (const auto & seq : alts) {
|
||||
auto saved = caps;
|
||||
if (match_seq(seq, i, end)) {
|
||||
return true;
|
||||
}
|
||||
caps = std::move(saved);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {
|
||||
size_t pos = i;
|
||||
for (const auto & it : seq) {
|
||||
int count = 0;
|
||||
while (count < it.max) {
|
||||
const bool is_group = it.atom.op == tts_norm_op::GROUP;
|
||||
std::vector<std::pair<int64_t, int64_t>> saved;
|
||||
if (is_group) {
|
||||
saved = caps;
|
||||
}
|
||||
size_t e = pos;
|
||||
if (!match_once(it.atom, pos, e)) {
|
||||
if (is_group) {
|
||||
caps = std::move(saved);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (e == pos) {
|
||||
// an empty iteration would repeat forever, it satisfies any remaining minimum
|
||||
count = std::max(count + 1, it.min);
|
||||
break;
|
||||
}
|
||||
pos = e;
|
||||
count++;
|
||||
}
|
||||
if (count < it.min) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
end = pos;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {
|
||||
const size_t n = t.size();
|
||||
switch (node.op) {
|
||||
case tts_norm_op::LITERAL:
|
||||
if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::SET:
|
||||
if (i < n && set_match(node.set, t[i])) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::ANY:
|
||||
if (i < n) {
|
||||
end = i + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case tts_norm_op::START:
|
||||
end = i;
|
||||
return i == 0;
|
||||
case tts_norm_op::END:
|
||||
end = i;
|
||||
return i == n;
|
||||
case tts_norm_op::BOUNDARY:
|
||||
end = i;
|
||||
return (i > 0 && is_w(i - 1)) != is_w(i);
|
||||
case tts_norm_op::GROUP: {
|
||||
size_t e = i;
|
||||
if (!match_alts(node.alts, i, e)) {
|
||||
return false;
|
||||
}
|
||||
if (node.capture >= 0) {
|
||||
caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};
|
||||
}
|
||||
end = e;
|
||||
return true;
|
||||
}
|
||||
case tts_norm_op::LOOKAHEAD: {
|
||||
// captures made inside a lookahead are discarded
|
||||
auto saved = caps;
|
||||
size_t e = i;
|
||||
const bool ok = match_alts(node.alts, i, e);
|
||||
caps = std::move(saved);
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
case tts_norm_op::LOOKBEHIND: {
|
||||
size_t e = 0;
|
||||
const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;
|
||||
end = i;
|
||||
return ok != node.neg;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// common_tts_norm_pattern
|
||||
//
|
||||
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {
|
||||
if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {
|
||||
throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");
|
||||
}
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(src, cpts, offs);
|
||||
|
||||
tts_norm_parser parser(std::move(cpts), n_caps);
|
||||
static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};
|
||||
if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {
|
||||
icase = true;
|
||||
parser.p = 4;
|
||||
}
|
||||
|
||||
root = std::make_unique<tts_norm_pattern_node>();
|
||||
root->op = tts_norm_op::GROUP;
|
||||
root->alts = parser.parse_alts();
|
||||
if (!parser.eof()) {
|
||||
parser.fail("unbalanced ')'");
|
||||
}
|
||||
}
|
||||
|
||||
common_tts_norm_pattern::~common_tts_norm_pattern() = default;
|
||||
common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;
|
||||
common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;
|
||||
|
||||
bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {
|
||||
tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};
|
||||
size_t e = pos;
|
||||
if (!m.match_alts(root->alts, pos, e)) {
|
||||
return false;
|
||||
}
|
||||
end = e;
|
||||
groups = std::move(m.caps);
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t common_tts_norm_pattern::n_groups() const {
|
||||
return n_caps;
|
||||
}
|
||||
|
||||
//
|
||||
// scan / sub
|
||||
//
|
||||
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {
|
||||
std::vector<uint32_t> cpts;
|
||||
std::vector<size_t> offs;
|
||||
tts_norm_decode_utf8(text, cpts, offs);
|
||||
|
||||
std::vector<common_tts_norm_segment> out;
|
||||
const size_t n = cpts.size();
|
||||
size_t i = 0;
|
||||
size_t pending = 0; // start of the current unmatched run
|
||||
std::vector<std::pair<int64_t, int64_t>> groups;
|
||||
|
||||
while (i < n) {
|
||||
bool matched = false;
|
||||
for (size_t r = 0; r < rules.size(); r++) {
|
||||
size_t end = i;
|
||||
if (rules[r]->match_at(cpts, i, end, groups) && end > i) {
|
||||
if (pending < i) {
|
||||
out.push_back({-1, offs[pending], offs[i], {}});
|
||||
}
|
||||
common_tts_norm_segment seg;
|
||||
seg.rule = static_cast<int>(r);
|
||||
seg.start = offs[i];
|
||||
seg.end = offs[end];
|
||||
for (const auto & g : groups) {
|
||||
if (g.first < 0) {
|
||||
seg.groups.emplace_back(std::string::npos, std::string::npos);
|
||||
} else {
|
||||
seg.groups.emplace_back(offs[g.first], offs[g.second]);
|
||||
}
|
||||
}
|
||||
out.push_back(std::move(seg));
|
||||
i = end;
|
||||
pending = i;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (pending < n) {
|
||||
out.push_back({-1, offs[pending], offs[n], {}});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
struct tts_norm_repl_piece {
|
||||
std::string literal;
|
||||
int group = -1; // 0 = whole match, 1.. = groups
|
||||
};
|
||||
|
||||
static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {
|
||||
std::vector<tts_norm_repl_piece> pieces;
|
||||
std::string lit;
|
||||
auto flush = [&]() {
|
||||
if (!lit.empty()) {
|
||||
pieces.push_back({lit, -1});
|
||||
lit.clear();
|
||||
}
|
||||
};
|
||||
for (size_t i = 0; i < repl.size(); i++) {
|
||||
if (repl[i] != '%') {
|
||||
lit += repl[i];
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= repl.size()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: dangling '%'");
|
||||
}
|
||||
const char c = repl[++i];
|
||||
if (c == '%') {
|
||||
lit += '%';
|
||||
} else if (c >= '0' && c <= '9') {
|
||||
const int g = c - '0';
|
||||
if (static_cast<size_t>(g) > pattern.n_groups()) {
|
||||
throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");
|
||||
}
|
||||
flush();
|
||||
pieces.push_back({"", g});
|
||||
} else {
|
||||
throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return pieces;
|
||||
}
|
||||
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {
|
||||
const auto pieces = tts_norm_parse_repl(repl, pattern);
|
||||
std::string out;
|
||||
for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {
|
||||
if (seg.rule < 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
continue;
|
||||
}
|
||||
for (const auto & piece : pieces) {
|
||||
if (piece.group < 0) {
|
||||
out += piece.literal;
|
||||
} else if (piece.group == 0) {
|
||||
out.append(text, seg.start, seg.end - seg.start);
|
||||
} else {
|
||||
const auto & g = seg.groups[piece.group - 1];
|
||||
if (g.first != std::string::npos) {
|
||||
out.append(text, g.first, g.second - g.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
//
|
||||
// template builtins
|
||||
//
|
||||
|
||||
// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs
|
||||
static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {
|
||||
std::vector<std::pair<std::string, std::string>> rules;
|
||||
if (is_val<value_string>(v)) {
|
||||
rules.emplace_back("match", v->as_string().str());
|
||||
} else if (is_val<value_object>(v)) {
|
||||
for (const auto & kv : v->as_ordered_object()) {
|
||||
if (!is_val<value_string>(kv.second)) {
|
||||
throw std::runtime_error("tts_norm_scan: rule pattern must be a string");
|
||||
}
|
||||
rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());
|
||||
}
|
||||
} else if (is_val<value_array>(v)) {
|
||||
for (const auto & item : v->as_array()) {
|
||||
if (!is_val<value_array>(item) || item->as_array().size() != 2
|
||||
|| !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {
|
||||
throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");
|
||||
}
|
||||
rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());
|
||||
}
|
||||
} else {
|
||||
throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_scan(const func_args & args) {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_val<value_string>(args.get_pos(0));
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const std::string text = text_val.str();
|
||||
const bool is_input = text_val.all_parts_are_input();
|
||||
|
||||
const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));
|
||||
std::vector<common_tts_norm_pattern> patterns;
|
||||
patterns.reserve(rule_srcs.size());
|
||||
std::vector<const common_tts_norm_pattern *> ptrs;
|
||||
for (const auto & r : rule_srcs) {
|
||||
patterns.emplace_back(r.second);
|
||||
}
|
||||
for (const auto & p : patterns) {
|
||||
ptrs.push_back(&p);
|
||||
}
|
||||
|
||||
auto mk_str = [&](size_t start, size_t end) -> value {
|
||||
return mk_val<value_string>(string(text.substr(start, end - start), is_input));
|
||||
};
|
||||
|
||||
auto out = mk_val<value_array>();
|
||||
for (const auto & seg : common_tts_norm_scan(text, ptrs)) {
|
||||
auto groups = mk_val<value_array>();
|
||||
for (const auto & g : seg.groups) {
|
||||
// groups that did not participate are empty strings, so they render as nothing and are falsy
|
||||
groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));
|
||||
}
|
||||
auto obj = mk_val<value_object>();
|
||||
obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));
|
||||
obj->insert("text", mk_str(seg.start, seg.end));
|
||||
obj->insert("groups", groups);
|
||||
out->push_back(obj);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static value tts_norm_builtin_sub(const func_args & args) {
|
||||
args.ensure_count(3, 3);
|
||||
args.ensure_vals<value_string, value_string, value_string>();
|
||||
const string text_val = args.get_pos(0)->as_string();
|
||||
const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());
|
||||
const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value) {
|
||||
const auto it = std::lower_bound(dict.keys.begin(), dict.keys.end(), key);
|
||||
if (it == dict.keys.end() || *it != key) {
|
||||
return false;
|
||||
}
|
||||
const size_t idx = it - dict.keys.begin();
|
||||
if (idx >= dict.values.size()) {
|
||||
throw std::runtime_error("tts_dict: malformed dictionary, a key has no value");
|
||||
}
|
||||
value = dict.values[idx];
|
||||
return true;
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts, common_tts_g2p g2p) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_vals<value_string, value_string>();
|
||||
const std::string name = args.get_pos(0)->as_string().str();
|
||||
const auto it = dicts ? dicts->find(name) : common_tts_dicts::const_iterator();
|
||||
if (!dicts || it == dicts->end()) {
|
||||
throw std::runtime_error("tts_dict: unknown dictionary '" + name + "'");
|
||||
}
|
||||
std::string doc;
|
||||
if (!common_tts_dict_find(it->second, args.get_pos(1)->as_string().str(), doc)) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return value_from_json(common_json::parse(doc), false);
|
||||
}));
|
||||
// none when the caller gives no letter-to-sound model, or the model has no pronunciation
|
||||
ctx.set_val("tts_g2p", mk_val<value_func>("tts_g2p", [g2p](const func_args & args) -> value {
|
||||
args.ensure_count(1, 1);
|
||||
args.ensure_vals<value_string>();
|
||||
const std::string out = g2p ? g2p(args.get_pos(0)->as_string().str()) : std::string();
|
||||
if (out.empty()) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return mk_val<value_string>(out);
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates
|
||||
//
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// functions exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
// tts_dict(name, key) -> value of key in a dictionary of the model, or none
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
}
|
||||
|
||||
struct tts_norm_pattern_node;
|
||||
|
||||
class common_tts_norm_pattern {
|
||||
public:
|
||||
// throws std::runtime_error on invalid pattern
|
||||
explicit common_tts_norm_pattern(const std::string & src);
|
||||
~common_tts_norm_pattern();
|
||||
common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;
|
||||
common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;
|
||||
|
||||
// match anchored at codepoint index pos; on success, end is the codepoint index after the match
|
||||
// groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate
|
||||
bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;
|
||||
|
||||
size_t n_groups() const;
|
||||
|
||||
private:
|
||||
std::unique_ptr<tts_norm_pattern_node> root;
|
||||
size_t n_caps = 0;
|
||||
bool icase = false;
|
||||
};
|
||||
|
||||
// segment produced by common_tts_norm_scan(); byte offsets into the input text
|
||||
struct common_tts_norm_segment {
|
||||
int rule = -1; // index of the matched rule, -1 for unmatched text
|
||||
size_t start = 0;
|
||||
size_t end = 0;
|
||||
std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate
|
||||
};
|
||||
|
||||
// split text into matched and unmatched segments
|
||||
// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped
|
||||
std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);
|
||||
|
||||
// replace every non-overlapping match of pattern with repl
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// a dictionary of the model, e.g. a pronunciation lexicon
|
||||
struct common_tts_dict {
|
||||
std::vector<std::string> keys; // sorted by their bytes
|
||||
std::vector<std::string> values; // the JSON document of each key
|
||||
};
|
||||
|
||||
using common_tts_dicts = std::map<std::string, common_tts_dict>;
|
||||
|
||||
// binary search of key in a dictionary; on success, value is set to the JSON document
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
|
||||
|
||||
// pronunciation of a word by a letter-to-sound model, as phonemes separated by spaces (e.g. CMUdict notation)
|
||||
// empty when there is none; the notation is the one of the model, its "tts" template knows it
|
||||
using common_tts_g2p = std::function<std::string(const std::string & word)>;
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr,
|
||||
common_tts_g2p g2p = nullptr);
|
||||
+129
-528
@@ -3,14 +3,12 @@
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "chat-tts-norm.h"
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "json.h"
|
||||
#include "log.h"
|
||||
#include "parsers/parsers.h"
|
||||
#include "sampling.h"
|
||||
|
||||
#include "jinja/value.h"
|
||||
#include "jinja/runtime.h"
|
||||
@@ -114,6 +112,38 @@ const char * common_chat_role_to_string(common_chat_role role) {
|
||||
return "";
|
||||
}
|
||||
|
||||
json common_chat_msg_delimiters::to_json() const {
|
||||
json result = json::array();
|
||||
for (const auto & d : delimiters) {
|
||||
result.push_back({
|
||||
{ "role", common_chat_role_to_string(d.role) },
|
||||
{ "delimiter", d.delimiter },
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {
|
||||
common_chat_msg_delimiters result;
|
||||
|
||||
if (!delimiters.is_array()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
result.delimiters.reserve(delimiters.size());
|
||||
for (const auto & d : delimiters) {
|
||||
if (!d.is_object()) {
|
||||
continue;
|
||||
}
|
||||
result.delimiters.push_back({
|
||||
common_chat_role_from_string(d.value("role", std::string())),
|
||||
d.value("delimiter", std::string()),
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {
|
||||
for (auto & d : delimiters) {
|
||||
d.tokens = common_tokenize(vocab, d.delimiter, false, true);
|
||||
@@ -191,17 +221,10 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
|
||||
} else {
|
||||
auto & parts = jmsg["content"] = json::array();
|
||||
for (const auto & part : content_parts) {
|
||||
if (part.type == "tool_reference") {
|
||||
parts.push_back({
|
||||
{"type", part.type},
|
||||
{"name", part.name},
|
||||
});
|
||||
} else {
|
||||
parts.push_back({
|
||||
{"type", part.type},
|
||||
{"text", part.text},
|
||||
});
|
||||
}
|
||||
parts.push_back({
|
||||
{"type", part.type},
|
||||
{"text", part.text},
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -317,8 +340,6 @@ struct common_chat_templates {
|
||||
bool has_explicit_template; // Model had builtin template or template overridden was specified.
|
||||
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
|
||||
std::unique_ptr<common_chat_template> template_tool_use;
|
||||
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
|
||||
std::shared_ptr<common_tts_dicts> tts_dicts; // dictionaries of the model for tts_dict(), by name
|
||||
};
|
||||
|
||||
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
|
||||
@@ -380,16 +401,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
|
||||
throw std::invalid_argument("Missing content part type: " + part.dump());
|
||||
}
|
||||
const auto & type = part.at("type");
|
||||
common_chat_msg_content_part msg_part;
|
||||
msg_part.type = type;
|
||||
if (type == "text" || type == "media_marker") {
|
||||
msg_part.text = part.at("text");
|
||||
} else if (type == "tool_reference") {
|
||||
// names a request tool, the template renders its definition
|
||||
msg_part.name = part.at("name");
|
||||
} else {
|
||||
if (type != "text" && type != "media_marker") {
|
||||
throw std::invalid_argument("Unsupported content part type: " + type.dump());
|
||||
}
|
||||
common_chat_msg_content_part msg_part;
|
||||
msg_part.type = type;
|
||||
msg_part.text = part.at("text");
|
||||
msg.content_parts.push_back(msg_part);
|
||||
}
|
||||
} else if (!content.is_null()) {
|
||||
@@ -442,41 +459,6 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
|
||||
if (message.contains("tool_call_id")) {
|
||||
msg.tool_call_id = message.at("tool_call_id");
|
||||
}
|
||||
{
|
||||
json extra = json::object();
|
||||
for (const auto & [k, v] : message.items()) {
|
||||
if (k != "role" && k != "content" && k != "tool_calls" && k != "reasoning_content" &&
|
||||
k != "name" && k != "tool_call_id") {
|
||||
extra[k] = v;
|
||||
}
|
||||
}
|
||||
if (has_tool_calls) {
|
||||
// per tool call: keys besides type/function/id, and function keys besides name/arguments
|
||||
json tc_extra = json::array();
|
||||
bool any = false;
|
||||
for (const auto & tool_call : message.at("tool_calls")) {
|
||||
json e = json::object();
|
||||
for (const auto & [k, v] : tool_call.items()) {
|
||||
if (k != "type" && k != "function" && k != "id") {
|
||||
e[k] = v;
|
||||
}
|
||||
}
|
||||
for (const auto & [k, v] : tool_call.at("function").items()) {
|
||||
if (k != "name" && k != "arguments") {
|
||||
e["function"][k] = v;
|
||||
}
|
||||
}
|
||||
any = any || !e.empty();
|
||||
tc_extra.push_back(e);
|
||||
}
|
||||
if (any) {
|
||||
extra["__tool_call_extras"] = tc_extra;
|
||||
}
|
||||
}
|
||||
if (!extra.empty()) {
|
||||
msg.extra_json = extra.dump();
|
||||
}
|
||||
}
|
||||
|
||||
msgs.push_back(msg);
|
||||
}
|
||||
@@ -565,50 +547,6 @@ static json render_message_to_json(const std::vector<common_chat_msg> & msgs, co
|
||||
return messages_inp_normalizer(c).normalize(messages);
|
||||
}
|
||||
|
||||
// DeepSeek V4.1 reads non-OpenAI message keys and joins content blocks with "\n\n" around images,
|
||||
// so keep media markers as separate text parts and restore the extra keys
|
||||
static json render_message_to_json_deepseek_v4_1(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {
|
||||
json messages = json::array();
|
||||
for (const auto & msg : msgs) {
|
||||
json jmsg;
|
||||
if (msg.contains_media() && msg.content.empty()) {
|
||||
common_chat_msg typed = msg;
|
||||
for (auto & part : typed.content_parts) {
|
||||
part.type = "text";
|
||||
}
|
||||
jmsg = typed.to_json_oaicompat(/* concat_typed_text= */ false);
|
||||
} else {
|
||||
jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
|
||||
}
|
||||
if (!msg.extra_json.empty()) {
|
||||
const json extra = json::parse(msg.extra_json);
|
||||
for (const auto & [k, v] : extra.items()) {
|
||||
if (k == "__tool_call_extras") {
|
||||
if (!jmsg.contains("tool_calls")) {
|
||||
continue;
|
||||
}
|
||||
auto & tcs = jmsg.at("tool_calls");
|
||||
for (size_t i = 0; i < tcs.size() && i < v.size(); i++) {
|
||||
for (const auto & [tk, tv] : v[i].items()) {
|
||||
if (tk == "function") {
|
||||
for (const auto & [fk, fv] : tv.items()) {
|
||||
tcs[i]["function"][fk] = fv;
|
||||
}
|
||||
} else {
|
||||
tcs[i][tk] = tv;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (!jmsg.contains(k)) {
|
||||
jmsg[k] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
messages.push_back(std::move(jmsg));
|
||||
}
|
||||
return messages_inp_normalizer(c).normalize(messages);
|
||||
}
|
||||
|
||||
// DEPRECATED: only used in tests
|
||||
json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {
|
||||
jinja::caps c;
|
||||
@@ -624,17 +562,13 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
|
||||
|
||||
auto result = json::array();
|
||||
for (const auto & tool : tools) {
|
||||
json function {
|
||||
{ "name", tool.name },
|
||||
{ "description", tool.description },
|
||||
{ "parameters", json::parse(tool.parameters) },
|
||||
};
|
||||
if (tool.defer_loading) {
|
||||
function["defer_loading"] = true;
|
||||
}
|
||||
result.push_back({
|
||||
{ "type", "function" },
|
||||
{ "function", function },
|
||||
{ "function", {
|
||||
{ "name", tool.name },
|
||||
{ "description", tool.description },
|
||||
{ "parameters", json::parse(tool.parameters) },
|
||||
}},
|
||||
});
|
||||
}
|
||||
return result;
|
||||
@@ -671,19 +605,10 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
|
||||
}
|
||||
|
||||
const auto & function = tool.at("function");
|
||||
|
||||
// "defer_loading" sits on the function, or on the tool next to "type"; the function wins
|
||||
const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
|
||||
: tool.value("defer_loading", json(false));
|
||||
if (!defer.is_boolean()) {
|
||||
throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
|
||||
}
|
||||
|
||||
result.push_back({
|
||||
/* .name = */ function.at("name"),
|
||||
/* .description = */ function.value("description", ""),
|
||||
/* .parameters = */ function.value("parameters", json::object()).dump(),
|
||||
/* .defer_loading = */ defer.get<bool>(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -695,11 +620,8 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
|
||||
}
|
||||
|
||||
common_chat_continuation common_chat_continuation_parse(const common_json & value) {
|
||||
if (value.is_null()) {
|
||||
return COMMON_CHAT_CONTINUATION_NONE;
|
||||
}
|
||||
if (value.is_boolean()) {
|
||||
return value.get<bool>() ? COMMON_CHAT_CONTINUATION_AUTO : COMMON_CHAT_CONTINUATION_NONE;
|
||||
if (value.is_boolean() && value.get<bool>()) {
|
||||
return COMMON_CHAT_CONTINUATION_AUTO;
|
||||
}
|
||||
if (value.is_string()) {
|
||||
auto value_str = value.get<std::string>();
|
||||
@@ -710,7 +632,7 @@ common_chat_continuation common_chat_continuation_parse(const common_json & valu
|
||||
return COMMON_CHAT_CONTINUATION_CONTENT;
|
||||
}
|
||||
}
|
||||
throw std::invalid_argument("Invalid continue_final_message: expected a boolean, \"content\" or \"reasoning_content\"");
|
||||
return COMMON_CHAT_CONTINUATION_NONE;
|
||||
}
|
||||
|
||||
bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
|
||||
@@ -753,7 +675,7 @@ std::string common_chat_format_single(const struct common_chat_templates * tmpls
|
||||
if (!past_msg.empty()) {
|
||||
inputs.messages = past_msg;
|
||||
inputs.add_generation_prompt = false;
|
||||
fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt.str();
|
||||
fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt;
|
||||
}
|
||||
std::ostringstream ss;
|
||||
// if the past_msg ends with a newline, we must preserve it in the formatted version
|
||||
@@ -763,7 +685,7 @@ std::string common_chat_format_single(const struct common_chat_templates * tmpls
|
||||
// format chat with new_msg
|
||||
inputs.messages.push_back(new_msg);
|
||||
inputs.add_generation_prompt = add_ass;
|
||||
auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt.str();
|
||||
auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt;
|
||||
// get the diff part
|
||||
ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());
|
||||
return ss.str();
|
||||
@@ -787,7 +709,7 @@ std::string common_chat_format_example(const struct common_chat_templates *
|
||||
add_simple_msg("user", "Hello");
|
||||
add_simple_msg("assistant", "Hi there");
|
||||
add_simple_msg("user", "How are you?");
|
||||
return common_chat_templates_apply(tmpls, inputs).prompt.str();
|
||||
return common_chat_templates_apply(tmpls, inputs).prompt;
|
||||
}
|
||||
|
||||
#define CHATML_TEMPLATE_SRC \
|
||||
@@ -806,119 +728,19 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
return tmpls->has_explicit_template;
|
||||
}
|
||||
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts,
|
||||
const common_tts_g2p & g2p = nullptr) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts, g2p);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
jinja::runtime runtime(ctx);
|
||||
return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
|
||||
}
|
||||
|
||||
// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
|
||||
static bool is_lfm3_template(const std::string & src) {
|
||||
return src.find("<additional_tools>") != std::string::npos;
|
||||
}
|
||||
|
||||
static std::string string_lower(std::string s) {
|
||||
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); });
|
||||
return s;
|
||||
}
|
||||
|
||||
// VibeVoice-ASR: default system prompt of the template written by the converter
|
||||
static bool is_vibevoice_asr_template(const std::string & src) {
|
||||
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
{ "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
|
||||
{ "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
|
||||
{ "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
|
||||
{ "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
|
||||
};
|
||||
const auto it = names.find(string_lower(language));
|
||||
return it != names.end() ? it->second : language;
|
||||
}
|
||||
|
||||
// LFM3-Audio: no language = auto detect
|
||||
static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
if (language.empty()) {
|
||||
return "Perform ASR.";
|
||||
}
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
|
||||
double audio_length, const std::string & context) {
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
|
||||
common_chat_prompt_preset asr_preset;
|
||||
asr_preset.system = "";
|
||||
asr_preset.user = "Transcribe audio to text";
|
||||
|
||||
if (chat_templates && chat_templates->template_default) {
|
||||
const std::string src = chat_templates->template_default->source();
|
||||
if (is_lfm2_template(src)) {
|
||||
asr_preset.system = "Perform ASR.";
|
||||
asr_preset.user = "";
|
||||
} else if (is_lfm3_template(src)) {
|
||||
asr_preset.system = lfm3_asr_system_prompt(language);
|
||||
asr_preset.user = "";
|
||||
asr_preset.has_language = true;
|
||||
} else if (is_vibevoice_asr_template(src)) {
|
||||
// without the audio length the model does not know where the audio ends and loops
|
||||
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
|
||||
if (context.empty()) {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
|
||||
} else {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
|
||||
asr_preset.has_context = true;
|
||||
}
|
||||
asr_preset.media_first = true;
|
||||
asr_preset.needs_audio_length = true;
|
||||
}
|
||||
if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
|
||||
asr_preset.system = "Perform ASR.";
|
||||
asr_preset.user = "";
|
||||
}
|
||||
|
||||
return asr_preset;
|
||||
}
|
||||
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language, const std::string & voice, const common_tts_g2p & g2p) {
|
||||
if (!chat_templates || !chat_templates->template_tts) {
|
||||
return "";
|
||||
}
|
||||
return common_chat_template_render(*chat_templates->template_tts, {
|
||||
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
|
||||
{"language", language},
|
||||
{"voice", voice},
|
||||
}, chat_templates->tts_dicts, g2p);
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
const std::string & language, const std::string & voice) {
|
||||
if (!chat_templates || !chat_templates->template_default || !is_lfm3_template(chat_templates->template_default->source())) {
|
||||
return "";
|
||||
}
|
||||
// same voices as the "tts" template
|
||||
const std::string v = string_lower(voice);
|
||||
const bool has_voice = v == "female" || v == "male";
|
||||
if (language.empty() && !has_voice) {
|
||||
return "";
|
||||
}
|
||||
std::string out = "Speak";
|
||||
if (!language.empty()) {
|
||||
out += " in " + lfm3_language_name(language);
|
||||
}
|
||||
if (has_voice) {
|
||||
out += " using a " + v + " voice";
|
||||
}
|
||||
return out + ".";
|
||||
}
|
||||
|
||||
std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
|
||||
if (!variant.empty()) {
|
||||
if (variant == "tool_use") {
|
||||
@@ -974,6 +796,18 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
"{%- if false %}");
|
||||
}
|
||||
|
||||
// TODO @aldehir : this is a temporary fix, pending Minja changes
|
||||
// Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664
|
||||
if (default_template_src.find("[TOOL_CALLS]") != std::string::npos
|
||||
// search for the error message and patch it
|
||||
&& default_template_src.find("if (message['content'] is none or") != std::string::npos) {
|
||||
string_replace_all(default_template_src,
|
||||
"{%- if (message['content'] is none or message['content'] == '' or "
|
||||
"message['content']|length == 0) and (message['tool_calls'] is not defined or "
|
||||
"message['tool_calls'] is none or message['tool_calls']|length == 0) %}",
|
||||
"{%- if false %}");
|
||||
}
|
||||
|
||||
std::string token_bos = bos_token_override;
|
||||
std::string token_eos = eos_token_override;
|
||||
bool add_bos = false;
|
||||
@@ -1017,35 +851,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// task templates of the model, kept with a chat template override
|
||||
if (model) {
|
||||
if (const char * src = llama_model_chat_template(model, "tts")) {
|
||||
try {
|
||||
tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// dictionaries for tts_dict(), listed by name in tokenizer.tts_dicts
|
||||
tmpls->tts_dicts = std::make_shared<common_tts_dicts>();
|
||||
for (int32_t i = 0; i < llama_model_meta_arr_n(model, "tokenizer.tts_dicts"); i++) {
|
||||
const std::string name = llama_model_meta_arr_str(model, "tokenizer.tts_dicts", i);
|
||||
const std::string key = "tokenizer.tts_dict." + name + ".keys";
|
||||
const std::string val_key = "tokenizer.tts_dict." + name + ".values";
|
||||
const int32_t n = llama_model_meta_arr_n(model, key.c_str());
|
||||
if (n < 0 || llama_model_meta_arr_n(model, val_key.c_str()) != n) {
|
||||
LOG_ERR("%s: the tts dictionary '%s' has no value for each of its keys (ignoring it)\n", __func__, name.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & dict = (*tmpls->tts_dicts)[name];
|
||||
dict.keys.reserve(n);
|
||||
dict.values.reserve(n);
|
||||
for (int32_t j = 0; j < n; j++) {
|
||||
dict.keys.emplace_back(llama_model_meta_arr_str(model, key.c_str(), j));
|
||||
dict.values.emplace_back(llama_model_meta_arr_str(model, val_key.c_str(), j));
|
||||
}
|
||||
}
|
||||
}
|
||||
return tmpls;
|
||||
}
|
||||
|
||||
@@ -1097,7 +902,7 @@ common_reasoning_format common_reasoning_format_from_name(const std::string & fo
|
||||
throw std::runtime_error("Unknown reasoning format: " + format);
|
||||
}
|
||||
|
||||
common_chat_prompt common_chat_template_direct_apply_impl(
|
||||
std::string common_chat_template_direct_apply_impl(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
const std::optional<json> & messages_override,
|
||||
@@ -1143,37 +948,24 @@ common_chat_prompt common_chat_template_direct_apply_impl(
|
||||
|
||||
jinja::global_from_json(ctx, inp, inputs.mark_input);
|
||||
|
||||
// bos/eos come from the model, not from the user
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
|
||||
// render
|
||||
jinja::runtime runtime(ctx);
|
||||
const jinja::value results = runtime.execute(tmpl.prog);
|
||||
auto parts = jinja::runtime::gather_string_parts(results);
|
||||
|
||||
common_chat_prompt result;
|
||||
for (auto & part : parts->val_str.parts) {
|
||||
result.parts.push_back({std::move(part.val), part.is_input});
|
||||
}
|
||||
std::string result = parts->as_string().str();
|
||||
|
||||
// TODO: improve this later
|
||||
if (inputs.add_bos && !result.parts.empty()) {
|
||||
auto & first = result.parts.front();
|
||||
if (!first.is_input && string_starts_with(first.text, tmpl.bos_token())) {
|
||||
first.text.erase(0, tmpl.bos_token().size());
|
||||
}
|
||||
if (inputs.add_bos && string_starts_with(result, tmpl.bos_token())) {
|
||||
result = result.substr(tmpl.bos_token().size());
|
||||
}
|
||||
if (inputs.add_eos && !result.parts.empty()) {
|
||||
auto & last = result.parts.back();
|
||||
if (!last.is_input && string_ends_with(last.text, tmpl.eos_token())) {
|
||||
last.text.resize(last.text.size() - tmpl.eos_token().size());
|
||||
}
|
||||
if (inputs.add_eos && string_ends_with(result, tmpl.eos_token())) {
|
||||
result = result.substr(0, result.size() - tmpl.eos_token().size());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
common_chat_prompt common_chat_template_direct_apply(
|
||||
std::string common_chat_template_direct_apply(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);
|
||||
@@ -1189,9 +981,9 @@ std::string common_chat_template_generation_prompt_impl(
|
||||
autoparser::generation_params params = inputs;
|
||||
params.add_generation_prompt = false;
|
||||
params.continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
|
||||
std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context).str();
|
||||
std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context);
|
||||
params.add_generation_prompt = true;
|
||||
std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context).str();
|
||||
std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context);
|
||||
|
||||
size_t prefix_len = 0;
|
||||
size_t min_size = std::min(no_gen_prompt.size(), gen_prompt.size());
|
||||
@@ -1295,61 +1087,41 @@ static json common_chat_extra_context() {
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static common_chat_params common_chat_params_init_lfm2_tokens(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
|
||||
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ true);
|
||||
}
|
||||
|
||||
static common_chat_params common_chat_params_init_lfm2_5(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
|
||||
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
|
||||
}
|
||||
|
||||
// Pick the dedicated handler for a template from its source, or null for the autoparser.
|
||||
// Order matters: the first match wins, and later checks assume the earlier ones did not match.
|
||||
static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
|
||||
std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
const common_chat_template & tmpl,
|
||||
const std::string & src,
|
||||
autoparser::generation_params & params) {
|
||||
// Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser
|
||||
// Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them
|
||||
if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos &&
|
||||
src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n");
|
||||
return common_chat_params_init_ministral_3;
|
||||
return common_chat_params_init_ministral_3(tmpl, params);
|
||||
}
|
||||
|
||||
// LLM-jp-4.1 - GPT-OSS dialect (spaces after special tokens, <|end|>-separated parallel calls)
|
||||
if (src.find("chat_format=llm-jp-harmony-v1") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: LLM-jp Harmony v1\n");
|
||||
return common_chat_params_init_llm_jp_harmony;
|
||||
}
|
||||
|
||||
// Harmony XML tool calls, checked before the GPT-OSS JSON calls
|
||||
if (src.find("<|channel|>") != std::string::npos && src.find("<|constrain|>xml") != std::string::npos &&
|
||||
src.find("rfl:parameter") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Harmony XML\n");
|
||||
return common_chat_params_init_harmony_xml;
|
||||
return common_chat_params_init_llm_jp_harmony(tmpl, params);
|
||||
}
|
||||
|
||||
// GPT-OSS - has unique channel-based structure that needs dedicated handler
|
||||
if (src.find("<|channel|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: GPT-OSS\n");
|
||||
return common_chat_params_init_gpt_oss;
|
||||
}
|
||||
|
||||
// Muse Spark shares the Muse recipient protocol and adds a commentary recipient.
|
||||
if (src.find("to=commentary") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Muse Spark\n");
|
||||
return common_chat_params_init_muse_spark;
|
||||
return common_chat_params_init_gpt_oss(tmpl, params);
|
||||
}
|
||||
|
||||
// Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.
|
||||
if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Muse Glimmer\n");
|
||||
return common_chat_params_init_muse_glimmer;
|
||||
return common_chat_params_init_muse_glimmer(tmpl, params);
|
||||
}
|
||||
|
||||
// Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}
|
||||
// Detection: template has ">>>all" for content and ">>>" prefix for tool calls
|
||||
if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Functionary v3.2\n");
|
||||
return common_chat_params_init_functionary_v3_2;
|
||||
return common_chat_params_init_functionary_v3_2(tmpl, params);
|
||||
}
|
||||
|
||||
// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>
|
||||
@@ -1357,14 +1129,14 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
if (src.find("<|tool_calls_section_begin|>") != std::string::npos &&
|
||||
src.find("<|tool_call_begin|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Kimi K2 Thinking\n");
|
||||
return common_chat_params_init_kimi_k2;
|
||||
return common_chat_params_init_kimi_k2(tmpl, params);
|
||||
}
|
||||
|
||||
// Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it
|
||||
if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&
|
||||
src.find("<|end_of_msg|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Kimi K3\n");
|
||||
return common_chat_params_init_kimi_k3;
|
||||
return common_chat_params_init_kimi_k3(tmpl, params);
|
||||
}
|
||||
|
||||
// K2 Horizon - <|ifm|im_start|> turns, <ifm|think*> reasoning picked by reasoning_effort and
|
||||
@@ -1372,7 +1144,7 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
if (src.find("<|ifm|im_start|>") != std::string::npos &&
|
||||
src.find("<ifm|tool_calls>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: K2 Horizon\n");
|
||||
return common_chat_params_init_k2_horizon;
|
||||
return common_chat_params_init_k2_horizon(tmpl, params);
|
||||
}
|
||||
|
||||
// Ling 3.0 / Bailing V3 - <role>X</role> sections with <arg_key>/<arg_value> tagged
|
||||
@@ -1380,7 +1152,7 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
if (src.find("<role>ASSISTANT</role>") != std::string::npos &&
|
||||
src.find("<arg_key>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n");
|
||||
return common_chat_params_init_ling3;
|
||||
return common_chat_params_init_ling3(tmpl, params);
|
||||
}
|
||||
|
||||
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
|
||||
@@ -1389,19 +1161,19 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
if (src.find("<|START_TEXT|>") != std::string::npos &&
|
||||
src.find("<|START_ACTION|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Cohere2 MoE\n");
|
||||
return common_chat_params_init_cohere2moe;
|
||||
return common_chat_params_init_cohere2moe(tmpl, params);
|
||||
}
|
||||
|
||||
if (is_lfm2_template(src)) {
|
||||
LOG_DBG("Using specialized template: LFM2\n");
|
||||
return common_chat_params_init_lfm2_tokens;
|
||||
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);
|
||||
}
|
||||
|
||||
// LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokens
|
||||
if (src.find("List of tools: [") != std::string::npos &&
|
||||
src.find("<|tool_list_start|>") == std::string::npos) {
|
||||
LOG_DBG("Using specialized template: LFM2.5\n");
|
||||
return common_chat_params_init_lfm2_5;
|
||||
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false);
|
||||
}
|
||||
|
||||
// GigaChatV3 format detection
|
||||
@@ -1409,7 +1181,7 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
src.find("<|message_sep|>") != std::string::npos &&
|
||||
src.find("<|function_call|>") == std::string::npos) {
|
||||
LOG_DBG("Using specialized template: GigaChatV3\n");
|
||||
return common_chat_params_init_gigachat_v3;
|
||||
return common_chat_params_init_gigachat_v3(tmpl, params);
|
||||
}
|
||||
|
||||
// MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
|
||||
@@ -1418,26 +1190,29 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
src.find("<tool_call>") != std::string::npos &&
|
||||
src.find("<invoke name=") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: MiniMax-M3\n");
|
||||
return common_chat_params_init_minimax_m3;
|
||||
return common_chat_params_init_minimax_m3(tmpl, params);
|
||||
}
|
||||
|
||||
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
|
||||
// The template source contains the token as a variable assignment, not as a literal in markup.
|
||||
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
|
||||
// V4.1 tag names have a leading space and its template does not name the dsml_token variable.
|
||||
if ((src.find("dsml_token") != std::string::npos &&
|
||||
src.find("DSML") != std::string::npos &&
|
||||
(src.find("function_calls") != std::string::npos ||
|
||||
src.find("tool_calls") != std::string::npos)) ||
|
||||
src.find("|DSML| calls>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: DeepSeek V3.2/V4/V4.1\n");
|
||||
return common_chat_params_init_deepseek_v3_2;
|
||||
if (src.find("dsml_token") != std::string::npos &&
|
||||
src.find("DSML") != std::string::npos &&
|
||||
(src.find("function_calls") != std::string::npos ||
|
||||
src.find("tool_calls") != std::string::npos)) {
|
||||
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
|
||||
return common_chat_params_init_deepseek_v3_2(tmpl, params);
|
||||
}
|
||||
|
||||
// Gemma4 format detection
|
||||
if (src.find("'<|tool_call>call:'") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Gemma4\n");
|
||||
return common_chat_params_init_gemma4;
|
||||
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
|
||||
// apply workarounds if using the older gemma4 templates
|
||||
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
|
||||
"Consider updating to the official template.\n", __func__);
|
||||
workaround::convert_tool_responses_gemma4(params.messages);
|
||||
}
|
||||
return common_chat_params_init_gemma4(tmpl, params);
|
||||
}
|
||||
|
||||
// MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>
|
||||
@@ -1445,14 +1220,14 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
src.find("<function name=\"") != std::string::npos &&
|
||||
src.find("<param name=\"") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: MiniCPM5\n");
|
||||
return common_chat_params_init_minicpm5;
|
||||
return common_chat_params_init_minicpm5(tmpl, params);
|
||||
}
|
||||
|
||||
// TranslateGemma - user content must follow a custom schema with language codes
|
||||
if (src.find("[source_lang_code]") != std::string::npos &&
|
||||
src.find("[target_lang_code]") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: TranslateGemma\n");
|
||||
return common_chat_params_init_translate_gemma;
|
||||
return common_chat_params_init_translate_gemma(tmpl, params);
|
||||
}
|
||||
|
||||
// Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash
|
||||
@@ -1462,57 +1237,10 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
// Exclude models that don't use \n between tags
|
||||
src.find("'<tool_call><function=' ~ tool_call.name ~ '>'") == std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Qwen3-Coder\n");
|
||||
return common_chat_params_init_qwen3_coder;
|
||||
return common_chat_params_init_qwen3_coder(tmpl, params);
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
if (is_inkling_template(src)) {
|
||||
LOG_DBG("Using specialized template: Inkling\n");
|
||||
return common_chat_params_init_inkling;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
common_chat_template::common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
|
||||
jinja::lexer lexer;
|
||||
auto lexer_res = lexer.tokenize(src);
|
||||
this->prog = jinja::parse_from_tokens(lexer_res);
|
||||
|
||||
this->src = lexer_res.source;
|
||||
this->bos_tok = bos_token;
|
||||
this->eos_tok = eos_token;
|
||||
|
||||
this->caps = jinja::caps_get(prog);
|
||||
// LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
|
||||
|
||||
this->params_init = common_chat_template_detect_params_init(this->src);
|
||||
if (this->params_init) {
|
||||
return;
|
||||
}
|
||||
|
||||
// The analysis depends only on the template, so run it once here instead of on every apply.
|
||||
// A failure is kept for apply to report, so a bad template still loads like it did before.
|
||||
try {
|
||||
analysis = std::make_unique<autoparser::autoparser>();
|
||||
analysis->analyze_template(*this);
|
||||
} catch (const std::exception & e) {
|
||||
analysis.reset();
|
||||
analysis_error = e.what();
|
||||
}
|
||||
}
|
||||
|
||||
common_chat_template::~common_chat_template() = default;
|
||||
common_chat_template::common_chat_template(common_chat_template &&) = default;
|
||||
common_chat_template & common_chat_template::operator=(common_chat_template &&) = default;
|
||||
|
||||
std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & params) {
|
||||
if (!tmpl.params_init) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return tmpl.params_init(tmpl, params);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls,
|
||||
@@ -1532,9 +1260,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
workaround::trim_all_content(trimmed_messages);
|
||||
messages_to_render = &trimmed_messages;
|
||||
}
|
||||
params.messages = src.find("|DSML| calls>") != std::string::npos ?
|
||||
render_message_to_json_deepseek_v4_1(*messages_to_render, tmpl.original_caps()) :
|
||||
render_message_to_json(*messages_to_render, tmpl.original_caps());
|
||||
params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps());
|
||||
params.tool_choice = inputs.tool_choice;
|
||||
params.reasoning_format = inputs.reasoning_format;
|
||||
params.enable_thinking = inputs.enable_thinking;
|
||||
@@ -1616,23 +1342,21 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
|
||||
auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
|
||||
return p.literal(data.generation_prompt) << p.content(p.rest());
|
||||
});
|
||||
data.parser = parser.save();
|
||||
return data;
|
||||
}
|
||||
|
||||
if (auto result = common_chat_try_specialized_template(tmpl, params)) {
|
||||
if (auto result = common_chat_try_specialized_template(tmpl, src, params)) {
|
||||
return *result;
|
||||
}
|
||||
|
||||
if (!tmpl.analysis) {
|
||||
throw std::invalid_argument("Unable to generate parser for this template. Automatic parser generation failed: " + tmpl.analysis_error);
|
||||
}
|
||||
|
||||
try {
|
||||
LOG_DBG("%s: using differential autoparser\n", __func__);
|
||||
const auto & autoparser = *tmpl.analysis;
|
||||
struct autoparser::autoparser autoparser;
|
||||
autoparser.analyze_template(tmpl);
|
||||
auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser);
|
||||
|
||||
common_chat_msg_delimiters delimiters;
|
||||
@@ -1653,7 +1377,8 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
auto_params.thinking_end_tags = {std::move(end_tag)};
|
||||
}
|
||||
}
|
||||
const auto & arena = auto_params.parser;
|
||||
common_peg_arena arena;
|
||||
arena.load(auto_params.parser);
|
||||
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
|
||||
return auto_params;
|
||||
} catch (const std::exception & e) {
|
||||
@@ -1718,7 +1443,7 @@ static common_chat_params common_chat_templates_apply_legacy(const struct common
|
||||
}
|
||||
|
||||
common_chat_params params;
|
||||
params.prompt = common_chat_prompt(std::string(buf.data(), res));
|
||||
params.prompt = std::string(buf.data(), res);
|
||||
if (!inputs.json_schema.empty()) {
|
||||
params.grammar = json_schema_to_grammar(json::parse(inputs.json_schema));
|
||||
} else {
|
||||
@@ -1734,46 +1459,6 @@ common_chat_params common_chat_templates_apply(const struct common_chat_template
|
||||
common_chat_templates_apply_legacy(tmpls, inputs);
|
||||
}
|
||||
|
||||
std::string common_chat_prompt::str() const {
|
||||
if (parts.size() == 1) {
|
||||
return parts[0].text;
|
||||
}
|
||||
size_t len = 0;
|
||||
for (const auto & part : parts) {
|
||||
len += part.text.size();
|
||||
}
|
||||
std::string out;
|
||||
out.reserve(len);
|
||||
for (const auto & part : parts) {
|
||||
out += part.text;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void common_chat_prompt::append(const std::string & text, bool is_input) {
|
||||
if (text.empty()) {
|
||||
return;
|
||||
}
|
||||
if (!parts.empty() && parts.back().is_input == is_input) {
|
||||
parts.back().text += text;
|
||||
} else {
|
||||
parts.push_back({text, is_input});
|
||||
}
|
||||
}
|
||||
|
||||
void common_chat_prompt::replace_last(const std::string & from, const std::string & to) {
|
||||
for (auto it = parts.rbegin(); it != parts.rend(); ++it) {
|
||||
if (it->is_input) {
|
||||
continue;
|
||||
}
|
||||
size_t pos = it->text.rfind(from);
|
||||
if (pos != std::string::npos) {
|
||||
it->text.replace(pos, from.size(), to);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void common_chat_input::append(const std::string & piece, llama_token token) {
|
||||
if (piece.empty()) {
|
||||
return;
|
||||
@@ -1840,10 +1525,9 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
const common_chat_input & input,
|
||||
bool is_partial,
|
||||
const common_chat_parser_params & params) {
|
||||
// both branches must be lvalues, a temporary here would copy the arena on every call
|
||||
static const common_peg_arena content_only =
|
||||
build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); });
|
||||
const common_peg_arena & parser = src_parser.empty() ? content_only : src_parser;
|
||||
const common_peg_arena & parser = src_parser.empty() ?
|
||||
build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }) :
|
||||
src_parser;
|
||||
|
||||
if (src_parser.empty()) {
|
||||
LOG_DBG("No parser definition detected, assuming pure content parser.");
|
||||
@@ -1855,9 +1539,6 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
|
||||
|
||||
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
|
||||
if (!is_partial) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_COMPLETE;
|
||||
}
|
||||
if (params.debug) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
|
||||
}
|
||||
@@ -1917,92 +1598,12 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
return msg;
|
||||
}
|
||||
|
||||
common_chat_session::common_chat_session(const common_chat_templates * tmpls,
|
||||
const llama_vocab * vocab,
|
||||
const common_chat_templates_inputs & inputs,
|
||||
const common_chat_session_params & params) {
|
||||
auto applied = common_chat_templates_apply(tmpls, inputs);
|
||||
|
||||
templated = true;
|
||||
prompt_parts = std::move(applied.prompt);
|
||||
result.role = "assistant";
|
||||
|
||||
grammar_text = std::move(applied.grammar);
|
||||
grammar_lazy = applied.grammar_lazy;
|
||||
stops = std::move(applied.additional_stops);
|
||||
generation_prompt_text = applied.generation_prompt;
|
||||
thinking_start = std::move(applied.thinking_start_tag);
|
||||
thinking_ends = std::move(applied.thinking_end_tags);
|
||||
|
||||
parser_params.format = applied.format;
|
||||
parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt)
|
||||
: common_chat_input(applied.generation_prompt);
|
||||
parser_params.debug = params.debug;
|
||||
parser_params.parser = std::move(applied.parser);
|
||||
|
||||
delimiters = std::move(applied.message_delimiters);
|
||||
|
||||
if (vocab) {
|
||||
common_params_sampling resolved;
|
||||
resolved.grammar_lazy = applied.grammar_lazy;
|
||||
common_sampling_add_preserved_tokens(resolved, vocab, applied.preserved_tokens);
|
||||
common_sampling_add_grammar_triggers(resolved, vocab, std::move(applied.grammar_triggers));
|
||||
preserved_tokens = std::move(resolved.preserved_tokens);
|
||||
grammar_triggers = std::move(resolved.grammar_triggers);
|
||||
|
||||
delimiters.tokenize(vocab);
|
||||
} else {
|
||||
grammar_triggers = std::move(applied.grammar_triggers);
|
||||
}
|
||||
|
||||
if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !params.echo) {
|
||||
// start from the prefill so it is not emitted as part of the first delta
|
||||
result = common_chat_parse(input, true, parser_params);
|
||||
}
|
||||
}
|
||||
|
||||
void common_chat_session::apply_sampling(common_params_sampling & sampling) const {
|
||||
if (!templated) {
|
||||
return;
|
||||
}
|
||||
if (!grammar_text.empty()) {
|
||||
sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, grammar_text};
|
||||
}
|
||||
sampling.grammar_lazy = grammar_lazy;
|
||||
sampling.generation_prompt = generation_prompt_text;
|
||||
sampling.preserved_tokens.insert(preserved_tokens.begin(), preserved_tokens.end());
|
||||
sampling.grammar_triggers.insert(sampling.grammar_triggers.end(), grammar_triggers.begin(), grammar_triggers.end());
|
||||
}
|
||||
|
||||
const common_chat_msg & common_chat_session::feed(const common_chat_input & chunk) {
|
||||
GGML_ASSERT(!finished && "feed() after finish()");
|
||||
input.append(chunk);
|
||||
auto msg = common_chat_parse(input, true, parser_params);
|
||||
if (!msg.empty()) {
|
||||
result = std::move(msg);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const common_chat_msg & common_chat_session::finish(const common_chat_input & chunk) {
|
||||
GGML_ASSERT(!finished && "finish() called twice");
|
||||
finished = true;
|
||||
input.append(chunk);
|
||||
auto msg = common_chat_parse(input, false, parser_params);
|
||||
if (!msg.empty()) {
|
||||
result = std::move(msg);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
|
||||
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
|
||||
GGML_ASSERT(chat_templates != nullptr);
|
||||
GGML_ASSERT(chat_templates->template_default != nullptr);
|
||||
// take the more expressive template when available
|
||||
const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
|
||||
: chat_templates->template_default->caps;
|
||||
common_json result = caps.to_map();
|
||||
result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
|
||||
result["reasoning_effort_rejected"] = caps.reasoning_effort_rejected;
|
||||
return result;
|
||||
if (chat_templates->template_tool_use != nullptr) {
|
||||
// take the more expressive template when available
|
||||
return chat_templates->template_tool_use->caps.to_map();
|
||||
}
|
||||
return chat_templates->template_default->caps.to_map();
|
||||
}
|
||||
+31
-129
@@ -22,16 +22,8 @@ struct common_chat_templates;
|
||||
|
||||
namespace autoparser {
|
||||
struct generation_params;
|
||||
struct autoparser;
|
||||
} // namespace autoparser
|
||||
|
||||
struct common_chat_params;
|
||||
struct common_chat_template;
|
||||
|
||||
// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)
|
||||
using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs);
|
||||
|
||||
struct common_chat_tool_call {
|
||||
std::string name;
|
||||
std::string arguments;
|
||||
@@ -45,7 +37,6 @@ struct common_chat_tool_call {
|
||||
struct common_chat_msg_content_part {
|
||||
std::string type;
|
||||
std::string text;
|
||||
std::string name; // tool_reference: name of a tool from the request tools
|
||||
|
||||
// TODO @ngxson : no known chat templates support reasoning_content in content parts yet
|
||||
// this can be useful for models with interleaved thinking (like Kimi-K2)
|
||||
@@ -53,7 +44,7 @@ struct common_chat_msg_content_part {
|
||||
// std::string reasoning_content;
|
||||
|
||||
bool operator==(const common_chat_msg_content_part & other) const {
|
||||
return type == other.type && text == other.text && name == other.name;
|
||||
return type == other.type && text == other.text;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -63,20 +54,19 @@ struct common_chat_template {
|
||||
std::string eos_tok;
|
||||
std::string src;
|
||||
chat_template_caps caps;
|
||||
// Dedicated handler picked once from the source, null when the differential autoparser is used
|
||||
common_chat_params_init_fn params_init = nullptr;
|
||||
|
||||
// Differential analysis, run once here when there is no dedicated handler. Null when there
|
||||
// is one, or when the analysis failed, in which case analysis_error says why.
|
||||
std::unique_ptr<autoparser::autoparser> analysis;
|
||||
std::string analysis_error;
|
||||
common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
|
||||
jinja::lexer lexer;
|
||||
auto lexer_res = lexer.tokenize(src);
|
||||
this->prog = jinja::parse_from_tokens(lexer_res);
|
||||
|
||||
common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);
|
||||
this->src = lexer_res.source;
|
||||
this->bos_tok = bos_token;
|
||||
this->eos_tok = eos_token;
|
||||
|
||||
// autoparser is incomplete here, so these are defined where it is complete
|
||||
~common_chat_template();
|
||||
common_chat_template(common_chat_template &&);
|
||||
common_chat_template & operator=(common_chat_template &&);
|
||||
this->caps = jinja::caps_get(prog);
|
||||
// LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
|
||||
}
|
||||
|
||||
const std::string & source() const { return src; }
|
||||
const std::string & bos_token() const { return bos_tok; }
|
||||
@@ -95,8 +85,6 @@ struct common_chat_msg {
|
||||
std::string reasoning_content;
|
||||
std::string tool_name;
|
||||
std::string tool_call_id;
|
||||
// non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
|
||||
std::string extra_json;
|
||||
|
||||
common_json to_json_oaicompat(bool concat_typed_text = false) const;
|
||||
|
||||
@@ -221,15 +209,14 @@ struct common_chat_msg_delimiters {
|
||||
|
||||
// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
|
||||
common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
|
||||
|
||||
common_json to_json() const;
|
||||
};
|
||||
|
||||
struct common_chat_tool {
|
||||
std::string name;
|
||||
std::string description;
|
||||
std::string parameters;
|
||||
|
||||
// withheld from the system prompt, a tool_reference reveals it later in the conversation
|
||||
bool defer_loading = false;
|
||||
};
|
||||
|
||||
enum common_chat_tool_choice {
|
||||
@@ -279,31 +266,9 @@ struct common_chat_templates_inputs {
|
||||
bool force_pure_content = false;
|
||||
};
|
||||
|
||||
struct common_chat_prompt_part {
|
||||
std::string text;
|
||||
bool is_input = false; // from user input, special tokens inside must not be parsed
|
||||
};
|
||||
|
||||
// rendered chat template, split into template parts and user input parts
|
||||
struct common_chat_prompt {
|
||||
std::vector<common_chat_prompt_part> parts;
|
||||
|
||||
common_chat_prompt() = default;
|
||||
|
||||
// single part of template text
|
||||
explicit common_chat_prompt(std::string text) { parts.push_back({std::move(text), false}); }
|
||||
|
||||
std::string str() const;
|
||||
|
||||
void append(const std::string & text, bool is_input = false);
|
||||
|
||||
// replace the last occurrence of `from`, user input parts are skipped
|
||||
void replace_last(const std::string & from, const std::string & to);
|
||||
};
|
||||
|
||||
struct common_chat_params {
|
||||
common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
common_chat_prompt prompt;
|
||||
std::string prompt;
|
||||
std::string grammar;
|
||||
bool grammar_lazy = false;
|
||||
std::string generation_prompt;
|
||||
@@ -313,7 +278,7 @@ struct common_chat_params {
|
||||
std::vector<common_grammar_trigger> grammar_triggers;
|
||||
std::vector<std::string> preserved_tokens;
|
||||
std::vector<std::string> additional_stops;
|
||||
common_peg_arena parser;
|
||||
std::string parser;
|
||||
common_chat_msg_delimiters message_delimiters;
|
||||
};
|
||||
|
||||
@@ -345,10 +310,16 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st
|
||||
// per-message parsing syntax
|
||||
// should be derived from common_chat_params
|
||||
struct common_chat_parser_params {
|
||||
common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
common_chat_input generation_prompt;
|
||||
bool debug = false; // Enable debug output for PEG parser
|
||||
common_peg_arena parser = {};
|
||||
common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
|
||||
// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
|
||||
bool reasoning_in_content = false;
|
||||
common_chat_input generation_prompt;
|
||||
bool parse_tool_calls = true;
|
||||
bool is_continuation = false;
|
||||
bool echo = false; // Include assistant prefilled msg in output
|
||||
bool debug = false; // Enable debug output for PEG parser
|
||||
common_peg_arena parser = {};
|
||||
common_chat_parser_params() = default;
|
||||
common_chat_parser_params(const common_chat_params & chat_params) {
|
||||
format = chat_params.format;
|
||||
@@ -394,60 +365,6 @@ const char * common_chat_format_name(common_chat_format format);
|
||||
common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
|
||||
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
|
||||
|
||||
struct common_chat_session_params {
|
||||
bool echo = false; // include the assistant prefill in the output when continuing a message
|
||||
bool debug = false; // enable debug output for the PEG parser
|
||||
};
|
||||
|
||||
class common_chat_session {
|
||||
public:
|
||||
common_chat_session() { result.role = "assistant"; }
|
||||
|
||||
common_chat_session(const common_chat_templates * tmpls,
|
||||
const llama_vocab * vocab,
|
||||
const common_chat_templates_inputs & inputs,
|
||||
const common_chat_session_params & params = {});
|
||||
|
||||
const common_chat_prompt & prompt() const { return prompt_parts; }
|
||||
common_chat_format format() const { return parser_params.format; }
|
||||
const common_chat_msg & msg() const { return result; }
|
||||
const common_peg_arena & parser() const { return parser_params.parser; }
|
||||
|
||||
const std::string & grammar() const { return grammar_text; }
|
||||
const std::string & generation_prompt() const { return generation_prompt_text; }
|
||||
const std::string & thinking_start_tag() const { return thinking_start; }
|
||||
const std::vector<std::string> & thinking_end_tags() const { return thinking_ends; }
|
||||
const std::vector<std::string> & additional_stops() const { return stops; }
|
||||
|
||||
const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }
|
||||
|
||||
void apply_sampling(common_params_sampling & sampling) const;
|
||||
|
||||
bool has_template() const { return templated; }
|
||||
|
||||
const common_chat_msg & feed(const common_chat_input & chunk);
|
||||
|
||||
const common_chat_msg & finish(const common_chat_input & chunk = {});
|
||||
|
||||
private:
|
||||
common_chat_prompt prompt_parts;
|
||||
std::string grammar_text;
|
||||
bool grammar_lazy = false;
|
||||
std::vector<common_grammar_trigger> grammar_triggers;
|
||||
std::set<llama_token> preserved_tokens;
|
||||
std::vector<std::string> stops;
|
||||
std::string generation_prompt_text;
|
||||
std::string thinking_start;
|
||||
std::vector<std::string> thinking_ends;
|
||||
|
||||
common_chat_parser_params parser_params;
|
||||
common_chat_msg_delimiters delimiters;
|
||||
common_chat_input input;
|
||||
common_chat_msg result;
|
||||
bool templated = false;
|
||||
bool finished = false;
|
||||
};
|
||||
|
||||
// used by arg and server
|
||||
const char * common_reasoning_format_name(common_reasoning_format format);
|
||||
common_reasoning_format common_reasoning_format_from_name(const std::string & format);
|
||||
@@ -472,9 +389,9 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
|
||||
common_json common_chat_tool_parameters(const common_json & function);
|
||||
|
||||
// get template caps, useful for reporting to server /props endpoint
|
||||
common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
|
||||
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
|
||||
|
||||
common_chat_prompt common_chat_template_direct_apply(
|
||||
std::string common_chat_template_direct_apply(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -484,31 +401,16 @@ std::string common_chat_template_generation_prompt(
|
||||
|
||||
std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & params);
|
||||
const std::string & src,
|
||||
autoparser::generation_params & params);
|
||||
|
||||
|
||||
// specialized per-task preset
|
||||
struct common_chat_prompt_preset {
|
||||
std::string system;
|
||||
std::string user;
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_context = false; // the user text already holds the context
|
||||
bool media_first = false; // the media goes before the user text
|
||||
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
|
||||
};
|
||||
|
||||
// audio_length: in seconds, 0 if unknown
|
||||
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
|
||||
double audio_length = 0.0, const std::string & context = "");
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
// g2p (optional): the letter-to-sound model of the template's tts_g2p(), e.g. mtmd_gen_audio_g2p()
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language = "", const std::string & voice = "",
|
||||
const std::function<std::string(const std::string & word)> & g2p = nullptr);
|
||||
|
||||
// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
const std::string & language, const std::string & voice);
|
||||
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);
|
||||
+149
-246
@@ -26,7 +26,6 @@
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <random>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
@@ -219,23 +218,6 @@ int32_t common_cpu_get_num_math() {
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(__riscv) && defined(__linux__)
|
||||
std::ifstream cpuinfo("/proc/cpuinfo");
|
||||
std::string line;
|
||||
int32_t n_a100 = 0;
|
||||
while (std::getline(cpuinfo, line)) {
|
||||
std::istringstream fields(line);
|
||||
std::string key;
|
||||
std::string separator;
|
||||
uint64_t marchid;
|
||||
if (fields >> key >> separator >> std::hex >> marchid &&
|
||||
key == "marchid" && separator == ":" && marchid == UINT64_C(0x8000000041000002)) {
|
||||
++n_a100;
|
||||
}
|
||||
}
|
||||
if (n_a100 > 0) {
|
||||
return n_a100;
|
||||
}
|
||||
#elif defined(__powerpc64__) || defined(__powerpc__)
|
||||
int32_t smt_factor = 1;
|
||||
int phy_cpus = common_cpu_get_num_physical_cores();
|
||||
@@ -759,56 +741,50 @@ static inline bool glob_class_match(const char c, const char * pattern, const ch
|
||||
}
|
||||
|
||||
// simple glob: * matches non-/ chars, ** matches anything including /, [] matches character class
|
||||
// only wildcards recurse, too many wildcards: no match
|
||||
static bool glob_match(const char * pattern, const char * str, int depth = 0) {
|
||||
if (depth >= COMMON_RECURSIVE_LIMIT) {
|
||||
static inline bool glob_match(const char * pattern, const char * str) {
|
||||
if (*pattern == '\0') {
|
||||
return *str == '\0';
|
||||
}
|
||||
if (pattern[0] == '*' && pattern[1] == '*') {
|
||||
const char * p = pattern + 2;
|
||||
if (glob_match(p, str)) return true;
|
||||
if (*str != '\0') return glob_match(pattern, str + 1);
|
||||
return false;
|
||||
}
|
||||
while (*pattern != '\0') {
|
||||
if (pattern[0] == '*' && pattern[1] == '*') {
|
||||
const char * p = pattern + 2;
|
||||
for (;; str++) {
|
||||
if (glob_match(p, str, depth + 1)) return true;
|
||||
if (*str == '\0') return false;
|
||||
}
|
||||
if (*pattern == '*') {
|
||||
const char * p = pattern + 1;
|
||||
for (; *str != '\0' && *str != '/'; str++) {
|
||||
if (glob_match(p, str)) return true;
|
||||
}
|
||||
if (*pattern == '*') {
|
||||
const char * p = pattern + 1;
|
||||
for (; *str != '\0' && *str != '/'; str++) {
|
||||
if (glob_match(p, str, depth + 1)) return true;
|
||||
}
|
||||
return glob_match(p, str, depth + 1);
|
||||
}
|
||||
if (*pattern == '?' && *str != '\0' && *str != '/') {
|
||||
pattern++;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
if (*pattern == '[') {
|
||||
const char * class_end = pattern + 1;
|
||||
// If first character after '[' is ']' or '-', treat it as literal
|
||||
if (*class_end == ']' || *class_end == '-') {
|
||||
class_end++;
|
||||
}
|
||||
while (*class_end != '\0' && *class_end != ']') {
|
||||
class_end++;
|
||||
}
|
||||
if (*class_end == ']') {
|
||||
if (*str == '\0' || !glob_class_match(*str, pattern + 1, class_end)) return false;
|
||||
pattern = class_end + 1;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
if (*str != '[') return false;
|
||||
pattern++;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
if (*pattern != *str) return false;
|
||||
pattern++;
|
||||
str++;
|
||||
return glob_match(p, str);
|
||||
}
|
||||
return *str == '\0';
|
||||
if (*pattern == '?' && *str != '\0' && *str != '/') {
|
||||
return glob_match(pattern + 1, str + 1);
|
||||
}
|
||||
if (*pattern == '[') {
|
||||
const char * class_end = pattern + 1;
|
||||
// If first character after '[' is ']' or '-', treat it as literal
|
||||
if (*class_end == ']' || *class_end == '-') {
|
||||
class_end++;
|
||||
}
|
||||
while (*class_end != '\0' && *class_end != ']') {
|
||||
class_end++;
|
||||
}
|
||||
if (*class_end == ']') {
|
||||
if (*str == '\0') return false;
|
||||
bool matched = glob_class_match(*str, pattern + 1, class_end);
|
||||
return matched && glob_match(class_end + 1, str + 1);
|
||||
} else {
|
||||
if (*str == '[') {
|
||||
return glob_match(pattern + 1, str + 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (*pattern == *str) {
|
||||
return glob_match(pattern + 1, str + 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool glob_match(const std::string & pattern, const std::string & str) {
|
||||
@@ -895,7 +871,27 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
|
||||
return true;
|
||||
}
|
||||
|
||||
#include <iostream>
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
std::wstring utf8_to_wstring(const std::string & str) {
|
||||
if (str.empty()) {
|
||||
return std::wstring();
|
||||
}
|
||||
|
||||
int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);
|
||||
|
||||
if (size <= 0) {
|
||||
return std::wstring();
|
||||
}
|
||||
|
||||
std::wstring wstr(size, 0);
|
||||
MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);
|
||||
|
||||
return wstr;
|
||||
}
|
||||
|
||||
std::string wstring_to_utf8(const std::wstring & str) {
|
||||
if (str.empty()) {
|
||||
return std::string();
|
||||
@@ -914,14 +910,16 @@ std::string wstring_to_utf8(const std::wstring & str) {
|
||||
}
|
||||
#endif
|
||||
|
||||
void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
|
||||
// a random suffix gives every writer its own temporary file, across processes and containers
|
||||
std::random_device rd;
|
||||
const uint64_t id = ((uint64_t) rd() << 32) | rd();
|
||||
// returns the path as a UTF-8 string, preserving its separators
|
||||
std::string fs_path_to_utf8(const std::filesystem::path & path) {
|
||||
const auto value = path.u8string();
|
||||
return std::string(value.begin(), value.end());
|
||||
}
|
||||
|
||||
void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
|
||||
std::error_code ec;
|
||||
std::filesystem::path path_tmp = path;
|
||||
path_tmp += "." + std::to_string(id) + ".tmp";
|
||||
path_tmp += ".tmp";
|
||||
|
||||
if (path.has_parent_path()) {
|
||||
std::filesystem::create_directories(path.parent_path(), ec);
|
||||
@@ -937,7 +935,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
|
||||
|
||||
if (file.fail() || ec) {
|
||||
std::filesystem::remove(path_tmp, ec);
|
||||
throw std::runtime_error("failed to write file: " + path.string());
|
||||
throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -963,9 +961,20 @@ void common_set_env(const std::string & name, const std::string & value) {
|
||||
#endif
|
||||
}
|
||||
|
||||
std::filesystem::path common_get_path_from_env(const std::string & name) {
|
||||
#if defined(_WIN32)
|
||||
const std::wstring wname = utf8_to_wstring(name);
|
||||
const wchar_t * wvalue = _wgetenv(wname.c_str());
|
||||
return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
|
||||
#else
|
||||
const char * value = std::getenv(name.c_str());
|
||||
return value ? std::filesystem::path(value) : std::filesystem::path();
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
static std::filesystem::path get_home_directory() {
|
||||
std::filesystem::path home = common_get_env("HOME");
|
||||
std::filesystem::path home = common_get_path_from_env("HOME");
|
||||
if (!home.empty()) {
|
||||
return home;
|
||||
}
|
||||
@@ -978,19 +987,19 @@ static std::filesystem::path get_home_directory() {
|
||||
#endif
|
||||
|
||||
std::filesystem::path fs_get_cache_directory() {
|
||||
std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE");
|
||||
std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
|
||||
if (!cache_directory.empty()) {
|
||||
return cache_directory;
|
||||
}
|
||||
#if defined(_WIN32)
|
||||
cache_directory = common_get_env("LOCALAPPDATA");
|
||||
cache_directory = common_get_path_from_env("LOCALAPPDATA");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
cache_directory = get_home_directory() / "Library/Caches";
|
||||
#else
|
||||
cache_directory = common_get_env("XDG_CACHE_HOME");
|
||||
cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
|
||||
if (cache_directory.empty()) {
|
||||
cache_directory = get_home_directory() / ".cache";
|
||||
}
|
||||
@@ -1001,12 +1010,12 @@ std::filesystem::path fs_get_cache_directory() {
|
||||
std::filesystem::path fs_get_config_directory() {
|
||||
std::filesystem::path config_directory;
|
||||
#if defined(_WIN32)
|
||||
config_directory = common_get_env("APPDATA");
|
||||
config_directory = common_get_path_from_env("APPDATA");
|
||||
if (config_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %APPDATA% directory");
|
||||
}
|
||||
#else
|
||||
config_directory = common_get_env("XDG_CONFIG_HOME");
|
||||
config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
|
||||
if (config_directory.empty()) {
|
||||
config_directory = get_home_directory() / ".config";
|
||||
}
|
||||
@@ -1020,9 +1029,9 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) {
|
||||
std::error_code ec;
|
||||
common_create_directories(cache_directory, ec);
|
||||
if (ec) {
|
||||
throw std::runtime_error("failed to create cache directory: " + cache_directory.string());
|
||||
throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
|
||||
}
|
||||
return cache_directory / filename;
|
||||
return cache_directory / std::filesystem::u8path(filename);
|
||||
}
|
||||
|
||||
//
|
||||
@@ -1161,16 +1170,8 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
|
||||
{ COMMON_DECISION_TYPE_LAYA, "laya" },
|
||||
{ COMMON_DECISION_TYPE_CLEF, "clef" },
|
||||
{ COMMON_DECISION_TYPE_PPLX_DECIDER, "pplx-decider" },
|
||||
{ COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" },
|
||||
{ COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" },
|
||||
{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
|
||||
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
|
||||
{ COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
|
||||
{ COMMON_DECISION_TYPE_H2O_LIGHTNING, "h2o-lightning" },
|
||||
{ COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
|
||||
{ COMMON_DECISION_TYPE_LFM3_D1, "lfm3-d1" },
|
||||
{ COMMON_DECISION_TYPE_LFM3_D1_OMNI, "lfm3-d1-omni" },
|
||||
// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
|
||||
};
|
||||
|
||||
static common_decision_type common_decision_type_from_string(const std::string & str) {
|
||||
@@ -1194,9 +1195,7 @@ common_decision_type common_get_decision_type(const struct llama_model * model)
|
||||
return common_decision_type_from_string(buf);
|
||||
}
|
||||
|
||||
common_gguf_info common_get_gguf_info(const std::string & fname) {
|
||||
common_gguf_info info;
|
||||
|
||||
common_decision_type common_get_decision_type(const std::string & fname) {
|
||||
struct gguf_init_params gguf_params = {
|
||||
/* .no_alloc = */ true,
|
||||
/* .ctx = */ nullptr,
|
||||
@@ -1204,73 +1203,31 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
|
||||
|
||||
gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));
|
||||
if (!gguf_ctx) {
|
||||
return info; // missing or unreadable file
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // missing or unreadable file
|
||||
}
|
||||
|
||||
std::string arch;
|
||||
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
|
||||
if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
|
||||
return info; // no architecture in the metadata
|
||||
if (arch_id < 0) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // no architecture in the metadata
|
||||
}
|
||||
const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
|
||||
if (gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
|
||||
}
|
||||
arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
|
||||
if (arch.empty()) {
|
||||
return info;
|
||||
return COMMON_DECISION_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
const int64_t type_id = gguf_find_key(gguf_ctx.get(), (arch + ".decision.type").c_str());
|
||||
const std::string key = arch + ".decision.type";
|
||||
const int64_t type_id = gguf_find_key(gguf_ctx.get(), key.c_str());
|
||||
if (type_id < 0) {
|
||||
info.decision_type = COMMON_DECISION_TYPE_NONE;
|
||||
} else if (gguf_get_kv_type(gguf_ctx.get(), type_id) == GGUF_TYPE_STRING) {
|
||||
info.decision_type = common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
|
||||
return COMMON_DECISION_TYPE_NONE;
|
||||
}
|
||||
|
||||
// same key and type as the model loader
|
||||
const int64_t ctx_id = gguf_find_key(gguf_ctx.get(), (arch + ".context_length").c_str());
|
||||
if (ctx_id >= 0 && gguf_get_kv_type(gguf_ctx.get(), ctx_id) == GGUF_TYPE_UINT32) {
|
||||
info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);
|
||||
if (gguf_get_kv_type(gguf_ctx.get(), type_id) != GGUF_TYPE_STRING) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
|
||||
}
|
||||
|
||||
// same key as llama_model_chat_template(model, nullptr)
|
||||
info.has_chat_tmpl = gguf_find_key(gguf_ctx.get(), "tokenizer.chat_template") >= 0;
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
void common_fit_apply(common_params & params, llama_model_params & mparams, llama_context_params & cparams) {
|
||||
if (!params.fit_params) {
|
||||
return;
|
||||
}
|
||||
|
||||
COM_TRC("%s", "fitting params to device memory ...\n");
|
||||
COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
|
||||
|
||||
// the draft context is created from the same base params and follows the main context, fit both together
|
||||
const bool has_draft = params.speculative.has_dft();
|
||||
const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
|
||||
common_params params_dft = common_base_params_to_speculative(params);
|
||||
|
||||
auto mparams_dft = common_model_params_to_llama(params_dft);
|
||||
auto cparams_dft = common_context_params_to_llama(params_dft);
|
||||
if (spec_mtp) {
|
||||
cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
|
||||
}
|
||||
cparams_dft.n_rs_seq = 0;
|
||||
|
||||
const common_fit_extra_model extra = {
|
||||
/*.paths_model =*/ params_dft.model.get_paths(),
|
||||
/*.mparams =*/ &mparams_dft,
|
||||
/*.cparams =*/ &cparams_dft,
|
||||
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
|
||||
};
|
||||
|
||||
common_fit_params(params.model.get_paths(), &mparams, &cparams,
|
||||
params.tensor_split,
|
||||
params.tensor_buft_overrides.data(),
|
||||
params.fit_params_target.data(),
|
||||
params.fit_params_min_ctx,
|
||||
has_draft || spec_mtp ? &extra : nullptr,
|
||||
params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
|
||||
return common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
|
||||
}
|
||||
|
||||
common_init_result::common_init_result(common_params & params, bool model_only) :
|
||||
@@ -1278,9 +1235,41 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
||||
common_fit_apply(params, mparams, cparams);
|
||||
if (params.fit_params) {
|
||||
COM_TRC("%s", "fitting params to device memory ...\n");
|
||||
COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
|
||||
|
||||
llama_model * model = common_model_load(params.model.get_paths(), mparams);
|
||||
// the draft context is created from the same base params and follows the main context, fit both together
|
||||
const bool has_draft = params.speculative.has_dft();
|
||||
const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
|
||||
common_params params_dft = common_base_params_to_speculative(params);
|
||||
|
||||
auto mparams_dft = common_model_params_to_llama(params_dft);
|
||||
auto cparams_dft = common_context_params_to_llama(params_dft);
|
||||
if (spec_mtp) {
|
||||
cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
|
||||
}
|
||||
cparams_dft.n_rs_seq = 0;
|
||||
|
||||
const common_fit_extra_model extra = {
|
||||
/*.path_model =*/ params_dft.model.path.c_str(),
|
||||
/*.mparams =*/ &mparams_dft,
|
||||
/*.cparams =*/ &cparams_dft,
|
||||
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
|
||||
};
|
||||
|
||||
common_fit_params(params.model.path.c_str(), &mparams, &cparams,
|
||||
params.tensor_split,
|
||||
params.tensor_buft_overrides.data(),
|
||||
params.fit_params_target.data(),
|
||||
params.fit_params_min_ctx,
|
||||
has_draft || spec_mtp ? &extra : nullptr,
|
||||
params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
|
||||
}
|
||||
|
||||
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
|
||||
if (model == NULL) {
|
||||
return;
|
||||
}
|
||||
@@ -1296,34 +1285,19 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
// these decision models return a score for each token via the embeddings output
|
||||
// TODO: maybe improve this in the future
|
||||
const auto decision_type = common_get_decision_type(model);
|
||||
// minicpm-ocr-jev reads its class scores from the pooled output, the pooling type is the one of the GGUF
|
||||
if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
|
||||
decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2 ||
|
||||
decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
|
||||
const auto pooling_type = decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV ? LLAMA_POOLING_TYPE_UNSPECIFIED : LLAMA_POOLING_TYPE_NONE;
|
||||
|
||||
decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
|
||||
params.embedding = true;
|
||||
params.pooling_type = pooling_type;
|
||||
params.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
|
||||
cparams.embeddings = true;
|
||||
cparams.pooling_type = pooling_type;
|
||||
cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
cparams.n_outputs_max = cparams.n_batch;
|
||||
cparams.n_outputs_max_per_seq = 1;
|
||||
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// 0 is n_batch
|
||||
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
|
||||
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
|
||||
}
|
||||
|
||||
if (llama_model_classifier_free_guidance(model)) {
|
||||
params.kv_unified = true;
|
||||
cparams.kv_unified = true;
|
||||
cparams.n_seq_max *= 2;
|
||||
}
|
||||
|
||||
// embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch
|
||||
// (server.cpp does this check for --embedding, but before the model is loaded)
|
||||
if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) {
|
||||
@@ -1426,14 +1400,6 @@ std::vector<llama_adapter_lora_ptr> & common_init_result::lora() {
|
||||
return pimpl->lora;
|
||||
}
|
||||
|
||||
// only for warmup and probe decodes, fill zeros as dummy input
|
||||
static void common_batch_set_zero_state(common_batch & batch, const llama_model * model, std::vector<float> & zeros) {
|
||||
zeros.assign(llama_model_n_embd_out(model), 0.0f);
|
||||
for (int32_t i = 0; i < batch.size(); ++i) {
|
||||
batch.set_embd_state(i, { zeros.data(), 1, zeros.size() });
|
||||
}
|
||||
}
|
||||
|
||||
common_init_result_ptr common_init_from_params(common_params & params, bool model_only) {
|
||||
common_init_result_ptr res(new common_init_result(params, model_only));
|
||||
|
||||
@@ -1464,7 +1430,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
|
||||
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);
|
||||
|
||||
const auto cvec = common_control_vector_load(params.control_vectors, llama_model_n_layer(model));
|
||||
const auto cvec = common_control_vector_load(params.control_vectors);
|
||||
if (cvec.n_embd == -1) {
|
||||
return res;
|
||||
}
|
||||
@@ -1540,8 +1506,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
if (llama_model_has_decoder(model)) {
|
||||
tmp.resize(std::min(tmp.size(), (size_t) params.n_batch));
|
||||
common_batch batch = common_batch_get_one(lctx, tmp);
|
||||
std::vector<float> zeros;
|
||||
common_batch_set_zero_state(batch, model, zeros);
|
||||
llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
|
||||
}
|
||||
llama_memory_clear(llama_get_memory(lctx), true);
|
||||
@@ -1609,8 +1573,6 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
|
||||
int ret;
|
||||
{
|
||||
common_batch batch = common_batch_get_one(ctx, tmp);
|
||||
std::vector<float> zeros;
|
||||
common_batch_set_zero_state(batch, llama_get_model(ctx), zeros);
|
||||
ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
|
||||
}
|
||||
if (ret != 0) {
|
||||
@@ -1727,19 +1689,6 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
|
||||
return mparams;
|
||||
}
|
||||
|
||||
struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams) {
|
||||
if (paths.size() == 1) {
|
||||
return llama_model_load_from_file(paths[0].c_str(), mparams);
|
||||
}
|
||||
|
||||
std::vector<const char *> c_paths;
|
||||
c_paths.reserve(paths.size());
|
||||
for (const auto & path : paths) {
|
||||
c_paths.push_back(path.c_str());
|
||||
}
|
||||
return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams);
|
||||
}
|
||||
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params) {
|
||||
auto cparams = llama_context_default_params();
|
||||
|
||||
@@ -1776,8 +1725,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
|
||||
cparams.type_k = params.cache_type_k;
|
||||
cparams.type_v = params.cache_type_v;
|
||||
|
||||
cparams.moe_cache_size = params.moe_cache_size;
|
||||
cparams.moe_cache_cpu_share = params.moe_cache_cpu_share;
|
||||
cparams.moe_cache_size = params.moe_cache_size;
|
||||
|
||||
return cparams;
|
||||
}
|
||||
@@ -2000,7 +1948,7 @@ float common_embd_similarity_cos(const float * embd1, const float * embd2, int n
|
||||
// Control vector utils
|
||||
//
|
||||
|
||||
static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info, int32_t n_layer) {
|
||||
static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info) {
|
||||
common_control_vector_data result = { -1, {} };
|
||||
|
||||
ggml_context * ctx = nullptr;
|
||||
@@ -2041,10 +1989,6 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str());
|
||||
result.n_embd = -1;
|
||||
break;
|
||||
} else if (layer_idx > n_layer) {
|
||||
COM_ERR("direction tensor layer index %d exceeds the model layer count %d in %s\n", layer_idx, n_layer, load_info.fname.c_str());
|
||||
result.n_embd = -1;
|
||||
break;
|
||||
}
|
||||
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
|
||||
@@ -2068,10 +2012,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
}
|
||||
|
||||
// extend if necessary - do not store data for layer 0 (it's not used)
|
||||
result.data.resize(std::max(result.data.size(), (size_t) result.n_embd * layer_idx), 0.0f);
|
||||
result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
|
||||
|
||||
const float * src = (const float *) tensor->data;
|
||||
float * dst = result.data.data() + (size_t) result.n_embd * (layer_idx - 1); // layer 1 at [0]
|
||||
float * dst = result.data.data() + result.n_embd * (layer_idx - 1); // layer 1 at [0]
|
||||
for (int j = 0; j < result.n_embd; j++) {
|
||||
dst[j] += src[j] * load_info.strength; // allows multiple directions for same layer in same file
|
||||
}
|
||||
@@ -2089,11 +2033,11 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
return result;
|
||||
}
|
||||
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer) {
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos) {
|
||||
common_control_vector_data result = { -1, {} };
|
||||
|
||||
for (const auto & info : load_infos) {
|
||||
auto cur = common_control_vector_load_one(info, n_layer);
|
||||
auto cur = common_control_vector_load_one(info);
|
||||
|
||||
if (cur.n_embd == -1) {
|
||||
result.n_embd = -1;
|
||||
@@ -2214,7 +2158,7 @@ void common_batch::clear() {
|
||||
}
|
||||
|
||||
int32_t common_batch::add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output) {
|
||||
tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} });
|
||||
tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, {} });
|
||||
return size() - 1;
|
||||
}
|
||||
|
||||
@@ -2252,34 +2196,8 @@ bool common_batch::set_embd(int32_t idx, llama_embd embd) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_batch::set_embd_state(int32_t idx, llama_embd state) {
|
||||
if (idx < 0 || idx >= size() || tokens[idx].state.data != nullptr) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].state = state;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_batch::set_embd_state_row(int32_t idx, int32_t row) {
|
||||
if (idx < 0 || idx >= size() || row < 0 || tokens[idx].state.data != nullptr || tokens[idx].state_row >= 0) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].state_row = row;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, { nullptr, 0, 0 }, {} };
|
||||
for (int32_t j = 0; j < n_pos; ++j) {
|
||||
t.pos[j] = pos[j];
|
||||
}
|
||||
tokens.push_back(t);
|
||||
return size() - 1;
|
||||
}
|
||||
|
||||
int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) {
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} };
|
||||
t.embd_row = row;
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, {} };
|
||||
for (int32_t j = 0; j < n_pos; ++j) {
|
||||
t.pos[j] = pos[j];
|
||||
}
|
||||
@@ -2307,12 +2225,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
|
||||
GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
|
||||
}
|
||||
} else if (t.embd_row >= 0) {
|
||||
idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row);
|
||||
if (idx < 0) {
|
||||
GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n);
|
||||
}
|
||||
llama_batch_ext_set_pos(res, idx, t.pos.data());
|
||||
} else {
|
||||
idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
|
||||
if (idx < 0) {
|
||||
@@ -2330,18 +2242,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.output) {
|
||||
llama_batch_ext_set_output_logits(res, idx, true);
|
||||
}
|
||||
if (t.state.data) {
|
||||
llama_batch_ext_set_embd_state(res, idx, t.state); // contexts without a state input ignore it
|
||||
}
|
||||
if (t.state_row >= 0 && !llama_batch_ext_set_embd_state_row(res, idx, t.state_row)) {
|
||||
GGML_ABORT("%s: failed to set the state row of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
|
||||
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
+28
-107
@@ -31,8 +31,6 @@
|
||||
#define DIRECTORY_SEPARATOR '/'
|
||||
#endif // _WIN32
|
||||
|
||||
#define COMMON_RECURSIVE_LIMIT 128
|
||||
|
||||
#define COM_DBG(fmt, ...) LOG_DBG("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)
|
||||
#define COM_TRC(fmt, ...) LOG_TRC("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)
|
||||
#define COM_INF(fmt, ...) LOG_INF("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)
|
||||
@@ -174,20 +172,18 @@ enum common_params_sampling_config : uint64_t {
|
||||
};
|
||||
|
||||
enum common_speculative_type {
|
||||
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2, // DFlash2 speculative decoding (DFlash + selector)
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR, // LiliCorr speculative decoding (DFlash + scorer)
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
|
||||
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MOD,
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
|
||||
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
|
||||
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
|
||||
};
|
||||
|
||||
// Grammar type enumeration
|
||||
@@ -313,18 +309,6 @@ struct common_params_model {
|
||||
std::string hf_file = ""; // HF file
|
||||
std::string docker_repo = ""; // Docker repo
|
||||
|
||||
std::vector<std::string> shards;
|
||||
|
||||
std::vector<std::string> get_paths() const {
|
||||
std::vector<std::string> paths = { path };
|
||||
for (const auto & shard : shards) {
|
||||
if (std::find(paths.begin(), paths.end(), shard) == paths.end()) {
|
||||
paths.push_back(shard);
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
std::string get_name() const {
|
||||
if (!hf_repo.empty()) {
|
||||
return hf_repo;
|
||||
@@ -359,8 +343,6 @@ struct common_params_speculative_draft {
|
||||
|
||||
int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default)
|
||||
|
||||
int32_t n_ubatch = 0; // physical batch size of the draft context (0 - use the target's n_ubatch)
|
||||
|
||||
ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K
|
||||
ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V
|
||||
|
||||
@@ -416,12 +398,7 @@ struct common_params_speculative {
|
||||
|
||||
uint32_t need_n_rs_seq() const {
|
||||
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2 ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR;
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
|
||||
});
|
||||
|
||||
return needs_rs_seq ? draft.n_max : 0u;
|
||||
@@ -616,8 +593,7 @@ struct common_params {
|
||||
ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K
|
||||
ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V
|
||||
|
||||
size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers
|
||||
float moe_cache_cpu_share = 0.0f; // share of the MoE cache misses computed on the CPU instead of uploaded
|
||||
size_t moe_cache_size = 0; // GPU cache size in bytes for the MoE experts kept in the CPU, split among the GPUs like the layers
|
||||
|
||||
common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO;
|
||||
|
||||
@@ -649,7 +625,7 @@ struct common_params {
|
||||
std::string cls_sep = "\t"; // separator of classification sequences
|
||||
|
||||
// server params
|
||||
int32_t port = 9931; // server listens on this network port
|
||||
int32_t port = 8080; // server listens on this network port
|
||||
bool reuse_port = false; // allow multiple sockets to bind to the same port
|
||||
int32_t timeout_read = 3600; // http read timeout in seconds
|
||||
int32_t timeout_write = timeout_read; // http write timeout in seconds
|
||||
@@ -662,13 +638,6 @@ struct common_params {
|
||||
int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset
|
||||
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
|
||||
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
|
||||
bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
|
||||
std::string cache_disk_path; // directory of the disk prompt cache, empty = disabled
|
||||
int32_t cache_disk_mib = 4096; // size limit of cache_disk_path in MiB, 0 = no limit
|
||||
|
||||
// llama-connect params
|
||||
bool server_connect = false;
|
||||
std::string server_connect_code = "";
|
||||
|
||||
std::string public_path = ""; // NOLINT
|
||||
std::string api_prefix = ""; // NOLINT
|
||||
@@ -704,7 +673,6 @@ struct common_params {
|
||||
// UI configs
|
||||
bool ui = true;
|
||||
bool ui_mcp_proxy = false;
|
||||
bool ui_mcp_proxy_allow_private_urls = false; // allow the MCP CORS proxy to reach local or private addresses
|
||||
std::string ui_config_json;
|
||||
|
||||
// "advanced" endpoints are disabled by default for better security
|
||||
@@ -715,7 +683,6 @@ struct common_params {
|
||||
// enable built-in tools
|
||||
std::vector<std::string> server_tools;
|
||||
std::string server_tools_runtime;
|
||||
bool server_tools_allow_unauth = false; // allow tools/MCP on a non-loopback address without API key
|
||||
|
||||
// MCP server configs (Cursor-compatible JSON)
|
||||
std::string mcp_servers_config; // path to JSON file with MCP server definitions
|
||||
@@ -725,7 +692,6 @@ struct common_params {
|
||||
std::string models_dir = ""; // directory containing models for the router server
|
||||
std::string models_preset = ""; // directory containing model presets for the router server
|
||||
int models_max = 4; // maximum number of models to load simultaneously
|
||||
int models_memory_margin = 0; // MiB of device memory to keep free when loading models (0 = no memory accounting)
|
||||
bool models_autoload = true; // automatically load models when requested via the router server
|
||||
std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty)
|
||||
|
||||
@@ -733,7 +699,6 @@ struct common_params {
|
||||
|
||||
std::string slot_save_path;
|
||||
std::string media_path; // path to directory for loading media files
|
||||
bool media_allow_private_urls = false; // allow http(s) media URLs on local or private addresses
|
||||
|
||||
float slot_prompt_similarity = 0.1f;
|
||||
|
||||
@@ -797,9 +762,7 @@ struct common_params {
|
||||
|
||||
// TTS params
|
||||
std::string tts_lang = "";
|
||||
std::string tts_voice = "";
|
||||
std::string tts_speaker_file = "";
|
||||
std::string tts_speaker_text = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
@@ -932,9 +895,13 @@ bool glob_match(const std::string & pattern, const std::string & str);
|
||||
//
|
||||
|
||||
#ifdef _WIN32
|
||||
std::string wstring_to_utf8(const std::wstring & str);
|
||||
std::wstring utf8_to_wstring(const std::string & str);
|
||||
std::string wstring_to_utf8(const std::wstring & str);
|
||||
#endif
|
||||
|
||||
// returns the path as a UTF-8 string, preserving its separators
|
||||
std::string fs_path_to_utf8(const std::filesystem::path & path);
|
||||
|
||||
//
|
||||
// Environment utils
|
||||
//
|
||||
@@ -944,6 +911,9 @@ std::string wstring_to_utf8(const std::wstring & str);
|
||||
std::string common_get_env(const std::string & name);
|
||||
void common_set_env(const std::string & name, const std::string & value);
|
||||
|
||||
// reads a path from the environment, an unset variable gives an empty path
|
||||
std::filesystem::path common_get_path_from_env(const std::string & name);
|
||||
|
||||
//
|
||||
// Filesystem utils
|
||||
//
|
||||
@@ -993,31 +963,16 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_LAYA, // score of one marker token per option, read from the embeddings output
|
||||
COMMON_DECISION_TYPE_CLEF, // all questions in one prompt, score of option i read from the embeddings output at row i
|
||||
COMMON_DECISION_TYPE_PPLX_DECIDER, // same as openjev, label codes of 1 or 2 letters
|
||||
COMMON_DECISION_TYPE_VLLM_DECISION2, // score of each option read from the embeddings output, the options of a question are marked in the batch
|
||||
COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention
|
||||
COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
|
||||
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
|
||||
COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
|
||||
COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
|
||||
COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
|
||||
COMMON_DECISION_TYPE_LFM3_D1, // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
|
||||
COMMON_DECISION_TYPE_LFM3_D1_OMNI, // same as lfm3-d1
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
// lfm3-d1: max questions packed in one prompt
|
||||
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// metadata of a GGUF file, read without loading the model
|
||||
struct common_gguf_info {
|
||||
common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid
|
||||
uint32_t n_ctx_train = 0; // 0 if unknown
|
||||
bool has_chat_tmpl = false; // has the default tokenizer.chat_template
|
||||
};
|
||||
|
||||
common_gguf_info common_get_gguf_info(const std::string & fname);
|
||||
// same as above, but reads a GGUF file; it does not load the model
|
||||
// returns COMMON_DECISION_TYPE_UNKNOWN if the file is missing, unreadable, or invalid
|
||||
common_decision_type common_get_decision_type(const std::string & fname);
|
||||
|
||||
// note: defines the model, context, samplers, ets. lifetimes
|
||||
struct common_init_result {
|
||||
@@ -1044,11 +999,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
struct llama_model_params common_model_params_to_llama ( common_params & params);
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params);
|
||||
|
||||
// apply --fit to mparams and cparams, no-op if params.fit_params is false
|
||||
void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
|
||||
|
||||
struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams);
|
||||
|
||||
// clear LoRA adapters from context, then apply new list of adapters
|
||||
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
|
||||
|
||||
@@ -1120,12 +1070,8 @@ struct common_batch {
|
||||
llama_seq_id seq_id; // the first sequence id, see add_seq()
|
||||
bool output;
|
||||
llama_embd embd; // non-owning view of the data passed to add_embd()/set_embd(), data == NULL if none
|
||||
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
|
||||
@@ -1161,20 +1107,10 @@ struct common_batch {
|
||||
// attach a token embedding to the entry at idx, can only be set once per entry
|
||||
bool set_embd(int32_t idx, llama_embd embd);
|
||||
|
||||
// attach a state embedding (e.g. the target hidden state for MTP) to the entry at idx, can only be set once per entry
|
||||
bool set_embd_state(int32_t idx, llama_embd state);
|
||||
|
||||
// the state of entry idx is row `row` of the h_nextn device buffer of the state source context, see llama_set_ctx_state_src()
|
||||
bool set_embd_state_row(int32_t idx, int32_t row);
|
||||
|
||||
// add an embedding-only entry (no token id)
|
||||
// pos points to n_pos positions
|
||||
int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
// add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other
|
||||
// pos points to n_pos positions
|
||||
int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
int32_t size() const { return (int32_t) tokens.size(); }
|
||||
};
|
||||
|
||||
@@ -1271,9 +1207,8 @@ struct common_control_vector_load_info {
|
||||
};
|
||||
|
||||
// Load control vectors, scale each by strength, and add them together.
|
||||
// Direction layer indices must be in [1, n_layer].
|
||||
// On error, returns {-1, empty}
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer);
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);
|
||||
|
||||
//
|
||||
// Split utils
|
||||
@@ -1325,20 +1260,6 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
|
||||
// prompt utils
|
||||
//
|
||||
|
||||
// leave default-inserted elements uninitialized; callers must fill them before use
|
||||
template <typename T>
|
||||
struct common_no_init_allocator : std::allocator<T> {
|
||||
template <typename U> struct rebind { using other = common_no_init_allocator<U>; };
|
||||
|
||||
common_no_init_allocator() = default;
|
||||
template <typename U> common_no_init_allocator(const common_no_init_allocator<U> &) {}
|
||||
|
||||
template <typename U> void construct(U * p) { ::new ((void *) p) U; }
|
||||
};
|
||||
|
||||
// sequence state data, filled by llama_state_seq_get_data_ext()
|
||||
using common_state_data = std::vector<uint8_t, common_no_init_allocator<uint8_t>>;
|
||||
|
||||
struct common_prompt_checkpoint {
|
||||
int64_t n_tokens;
|
||||
|
||||
@@ -1348,8 +1269,8 @@ struct common_prompt_checkpoint {
|
||||
llama_pos pos_min;
|
||||
llama_pos pos_max;
|
||||
|
||||
common_state_data data_tgt;
|
||||
common_state_data data_dft;
|
||||
std::vector<uint8_t> data_tgt;
|
||||
std::vector<uint8_t> data_dft;
|
||||
|
||||
// (optional) speculative-decoding implementation state stashed with the checkpoint
|
||||
// (e.g. eagle3's deferred-boundary g_embd row)
|
||||
|
||||
+118
-302
@@ -4,7 +4,6 @@
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include "download.h"
|
||||
#include "file-lock.h"
|
||||
#include "hf-cache.h"
|
||||
#include "json.h"
|
||||
|
||||
@@ -40,12 +39,9 @@
|
||||
// downloader
|
||||
//
|
||||
|
||||
// the suffix of the file a download writes until it completes
|
||||
static const std::string PARTIAL_SUFFIX = ".downloadInProgress";
|
||||
|
||||
static void write_etag(const std::string & path, const std::string & etag) {
|
||||
const std::string etag_path = path + ".etag";
|
||||
fs_write_atomic(etag_path, etag);
|
||||
fs_write_atomic(std::filesystem::u8path(etag_path), etag);
|
||||
LOG_DBG("%s: file etag saved: %s\n", __func__, etag_path.c_str());
|
||||
}
|
||||
|
||||
@@ -78,18 +74,6 @@ std::pair<std::string, std::string> common_download_split_repo_tag(const std::st
|
||||
return {hf_repo, tag};
|
||||
}
|
||||
|
||||
// file name from url, without the query string
|
||||
static std::string url_get_filename(const std::string & url) {
|
||||
std::string filename = url;
|
||||
if (auto pos = filename.rfind('/'); pos != std::string::npos) {
|
||||
filename = filename.substr(pos + 1);
|
||||
}
|
||||
if (auto pos = filename.find('?'); pos != std::string::npos) {
|
||||
filename = filename.substr(0, pos);
|
||||
}
|
||||
return filename;
|
||||
}
|
||||
|
||||
class ProgressBar : public common_download_callback {
|
||||
static inline std::mutex mutex;
|
||||
static inline std::map<const ProgressBar *, int> lines;
|
||||
@@ -113,8 +97,14 @@ public:
|
||||
ProgressBar() = default;
|
||||
|
||||
void on_start(const common_download_progress & p) override {
|
||||
filename = url_get_filename(p.url);
|
||||
filename = p.url;
|
||||
|
||||
if (auto pos = filename.rfind('/'); pos != std::string::npos) {
|
||||
filename = filename.substr(pos + 1);
|
||||
}
|
||||
if (auto pos = filename.find('?'); pos != std::string::npos) {
|
||||
filename = filename.substr(0, pos);
|
||||
}
|
||||
for (size_t i = 0; i < filename.size(); ++i) {
|
||||
if ((filename[i] & 0xC0) != 0x80) {
|
||||
if (len++ == 39) {
|
||||
@@ -196,13 +186,6 @@ static bool common_pull_file(httplib::Client & cli,
|
||||
[&](const httplib::Response &response) {
|
||||
if (p.downloaded > 0 && response.status != 206) {
|
||||
LOG_WRN("%s: server did not respond with 206 Partial Content for a resume request. Status: %d\n", func, response.status);
|
||||
if (response.status == 416 || response.status == 200) {
|
||||
// the kept partial cannot be resumed: 416 for a complete or stale one,
|
||||
// 200 from a server that ignores the range; drop it, so the next
|
||||
// attempt starts over instead of wedging on the same answer
|
||||
ofs.close();
|
||||
remove(path_tmp.c_str());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (p.downloaded == 0 && response.status != 200) {
|
||||
@@ -231,7 +214,7 @@ static bool common_pull_file(httplib::Client & cli,
|
||||
if (progress_step >= p.total / 1000 || p.downloaded == p.total) {
|
||||
if (callback) {
|
||||
callback->on_update(p);
|
||||
if (callback->is_cancelled() && (p.total == 0 || p.downloaded != p.total)) {
|
||||
if (callback->is_cancelled()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -264,61 +247,15 @@ static bool common_pull_file(httplib::Client & cli,
|
||||
static int common_download_file_single_online(const std::string & url,
|
||||
const std::string & path,
|
||||
const common_download_opts & opts,
|
||||
bool skip_etag,
|
||||
const std::string & lock_path,
|
||||
const std::string & alt_path) {
|
||||
bool skip_etag) {
|
||||
static const int max_attempts = 3;
|
||||
static const int retry_delay_seconds = 2;
|
||||
|
||||
const std::string path_temporary = path + PARTIAL_SUFFIX;
|
||||
const std::string path_lock = lock_path.empty() ? path + ".lock" : lock_path;
|
||||
|
||||
common_download_progress p;
|
||||
p.url = url;
|
||||
|
||||
if (opts.callback) {
|
||||
opts.callback->on_start(p);
|
||||
}
|
||||
|
||||
// the completion callback may itself take the same lock (the HF cache
|
||||
// finalizes blobs under it), so every exit goes through finish()
|
||||
common_file_lock lock(path_lock);
|
||||
auto finish = [&](int status) {
|
||||
lock.close();
|
||||
if (opts.callback) {
|
||||
opts.callback->on_done(p, is_http_status_ok(status));
|
||||
}
|
||||
return status;
|
||||
};
|
||||
|
||||
// alt_path is the HF snapshot: the blob may have been moved there when symlinks are unavailable
|
||||
auto cache_hit = [&] {
|
||||
return std::filesystem::exists(path) || (!alt_path.empty() && std::filesystem::exists(alt_path));
|
||||
};
|
||||
|
||||
if (skip_etag && cache_hit()) {
|
||||
LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
|
||||
return finish(304); // 304 Not Modified - fake cached response
|
||||
}
|
||||
|
||||
// take the lock before touching the cache, so two processes cannot race on the same file
|
||||
bool waited = false;
|
||||
if (!lock.acquire([&] {
|
||||
if (!waited) {
|
||||
LOG_INF("%s: file '%s' is being downloaded by another process, waiting...\n", __func__, url_get_filename(url).c_str());
|
||||
waited = true;
|
||||
}
|
||||
return !(opts.callback && opts.callback->is_cancelled());
|
||||
})) {
|
||||
return finish(-1);
|
||||
}
|
||||
|
||||
// under the lock: another process may have finished while we waited
|
||||
bool file_exists = cache_hit();
|
||||
const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path));
|
||||
|
||||
if (file_exists && skip_etag) {
|
||||
LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
|
||||
return finish(304); // 304 Not Modified - fake cached response
|
||||
return 304; // 304 Not Modified - fake cached response
|
||||
}
|
||||
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
@@ -347,9 +284,9 @@ static int common_download_file_single_online(const std::string & url,
|
||||
LOG_TRC("%s: HEAD failed, status: %d\n", __func__, head ? head->status : -1);
|
||||
if (file_exists) {
|
||||
LOG_TRC("%s: using cached file (HEAD failed): %s\n", __func__, path.c_str());
|
||||
return finish(304); // 304 Not Modified - fake cached response
|
||||
return 304; // 304 Not Modified - fake cached response
|
||||
}
|
||||
return finish(head ? head->status : -1);
|
||||
return head ? head->status : -1;
|
||||
}
|
||||
|
||||
std::string etag;
|
||||
@@ -357,6 +294,8 @@ static int common_download_file_single_online(const std::string & url,
|
||||
etag = head->get_header_value("ETag");
|
||||
}
|
||||
|
||||
common_download_progress p;
|
||||
p.url = url;
|
||||
if (head->has_header("Content-Length")) {
|
||||
try {
|
||||
p.total = std::stoull(head->get_header_value("Content-Length"));
|
||||
@@ -373,16 +312,16 @@ static int common_download_file_single_online(const std::string & url,
|
||||
if (file_exists) {
|
||||
if (etag.empty()) {
|
||||
LOG_DBG("%s: using cached file (no server etag): %s\n", __func__, path.c_str());
|
||||
return finish(304); // 304 Not Modified - fake cached response
|
||||
return 304; // 304 Not Modified - fake cached response
|
||||
}
|
||||
if (!last_etag.empty() && last_etag == etag) {
|
||||
LOG_DBG("%s: using cached file (same etag): %s\n", __func__, path.c_str());
|
||||
return finish(304); // 304 Not Modified - fake cached response
|
||||
return 304; // 304 Not Modified - fake cached response
|
||||
}
|
||||
// pass this point, the file exists but is different from the server version, so we need to redownload it
|
||||
if (remove(path.c_str()) != 0) {
|
||||
LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());
|
||||
return finish(-1);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -392,24 +331,11 @@ static int common_download_file_single_online(const std::string & url,
|
||||
}
|
||||
|
||||
bool success = false;
|
||||
const std::string path_temporary = path + ".downloadInProgress";
|
||||
int delay = retry_delay_seconds;
|
||||
|
||||
if (!skip_etag && !etag.empty()) {
|
||||
if (std::filesystem::exists(path_temporary) && read_etag(path_temporary) != etag) {
|
||||
LOG_WRN("%s: partial file does not match the server etag, discarding it: %s\n", __func__, path_temporary.c_str());
|
||||
if (remove(path_temporary.c_str()) != 0) {
|
||||
LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str());
|
||||
return finish(-1);
|
||||
}
|
||||
}
|
||||
write_etag(path_temporary, etag);
|
||||
}
|
||||
|
||||
if (opts.callback) {
|
||||
// report the resume offset, so a resumed file does not start at 0
|
||||
p.downloaded = supports_ranges && std::filesystem::exists(path_temporary) ?
|
||||
std::filesystem::file_size(path_temporary) : 0;
|
||||
opts.callback->on_update(p);
|
||||
opts.callback->on_start(p);
|
||||
}
|
||||
|
||||
for (int i = 0; i < max_attempts; ++i) {
|
||||
@@ -446,88 +372,77 @@ static int common_download_file_single_online(const std::string & url,
|
||||
}
|
||||
if (!etag.empty() && !skip_etag) {
|
||||
write_etag(path, etag);
|
||||
remove((path_temporary + ".etag").c_str());
|
||||
}
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// a stopped download keeps path_temporary, so the next attempt resumes from it
|
||||
if (opts.callback) {
|
||||
opts.callback->on_done(p, success);
|
||||
}
|
||||
if (opts.callback && opts.callback->is_cancelled() &&
|
||||
std::filesystem::exists(path_temporary)) {
|
||||
if (remove(path_temporary.c_str()) != 0) {
|
||||
LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, path_temporary.c_str());
|
||||
}
|
||||
}
|
||||
if (!success) {
|
||||
LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
|
||||
return -1; // max attempts reached
|
||||
}
|
||||
|
||||
return finish(success ? head->status : -1);
|
||||
return head->status;
|
||||
}
|
||||
|
||||
std::pair<long, std::vector<char>> common_remote_get_content(const std::string & url_in,
|
||||
std::pair<long, std::vector<char>> common_remote_get_content(const std::string & url,
|
||||
const common_remote_params & params) {
|
||||
static constexpr int max_redirects = 5;
|
||||
std::string url = url_in;
|
||||
for (int n_redirects = 0; ; n_redirects++) {
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
|
||||
httplib::Headers headers;
|
||||
for (const auto & h : params.headers) {
|
||||
headers.emplace(h.first, h.second);
|
||||
}
|
||||
if (headers.find("User-Agent") == headers.end()) {
|
||||
headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info()));
|
||||
}
|
||||
|
||||
if (params.timeout > 0) {
|
||||
cli.set_read_timeout(params.timeout, 0);
|
||||
cli.set_write_timeout(params.timeout, 0);
|
||||
}
|
||||
if (params.resolve_host) {
|
||||
// pin checked address, follow redirects manually to check each hop
|
||||
cli.set_direct_hostname_addr_map({{parts.host, params.resolve_host(parts.host)}});
|
||||
cli.set_follow_location(false);
|
||||
}
|
||||
|
||||
std::vector<char> buf;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char *data, size_t len) {
|
||||
buf.insert(buf.end(), data, data + len);
|
||||
return params.max_size == 0 ||
|
||||
buf.size() <= static_cast<size_t>(params.max_size);
|
||||
},
|
||||
nullptr
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw std::runtime_error("error: cannot make GET request");
|
||||
}
|
||||
|
||||
if (params.resolve_host && res->status >= 300 && res->status < 400 && res->has_header("Location")) {
|
||||
if (n_redirects >= max_redirects) {
|
||||
throw std::runtime_error("error: too many redirects");
|
||||
}
|
||||
url = common_http_resolve_location(url, res->get_header_value("Location"));
|
||||
continue;
|
||||
}
|
||||
|
||||
return { res->status, std::move(buf) };
|
||||
httplib::Headers headers;
|
||||
for (const auto & h : params.headers) {
|
||||
headers.emplace(h.first, h.second);
|
||||
}
|
||||
if (headers.find("User-Agent") == headers.end()) {
|
||||
headers.emplace("User-Agent", "llama-cpp/" + std::string(llama_build_info()));
|
||||
}
|
||||
|
||||
if (params.timeout > 0) {
|
||||
cli.set_read_timeout(params.timeout, 0);
|
||||
cli.set_write_timeout(params.timeout, 0);
|
||||
}
|
||||
|
||||
std::vector<char> buf;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char *data, size_t len) {
|
||||
buf.insert(buf.end(), data, data + len);
|
||||
return params.max_size == 0 ||
|
||||
buf.size() <= static_cast<size_t>(params.max_size);
|
||||
},
|
||||
nullptr
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw std::runtime_error("error: cannot make GET request");
|
||||
}
|
||||
|
||||
return { res->status, std::move(buf) };
|
||||
}
|
||||
|
||||
int common_download_file_single(const std::string & url,
|
||||
const std::string & path,
|
||||
const common_download_opts & opts,
|
||||
bool skip_etag,
|
||||
const std::string & lock_path,
|
||||
const std::string & alt_path) {
|
||||
bool skip_etag) {
|
||||
if (!opts.offline) {
|
||||
ProgressBar tty_cb;
|
||||
common_download_opts online_opts = opts;
|
||||
if (!online_opts.callback) {
|
||||
online_opts.callback = &tty_cb;
|
||||
}
|
||||
return common_download_file_single_online(url, path, online_opts, skip_etag, lock_path, alt_path);
|
||||
return common_download_file_single_online(url, path, online_opts, skip_etag);
|
||||
}
|
||||
|
||||
if (!std::filesystem::exists(path)) {
|
||||
if (!std::filesystem::exists(std::filesystem::u8path(path))) {
|
||||
LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
|
||||
return -1;
|
||||
}
|
||||
@@ -679,65 +594,33 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,
|
||||
return best;
|
||||
}
|
||||
|
||||
static const std::vector<common_download_sidecar> sidecars = {
|
||||
{ "mmproj", "mmproj", false, &common_download_opts::download_mmproj, &common_download_hf_plan::mmproj },
|
||||
{ "mtp", "mtp-", true, &common_download_opts::download_mtp, &common_download_hf_plan::mtp },
|
||||
{ "dflash", "dflash-", true, &common_download_opts::download_dflash, &common_download_hf_plan::dflash },
|
||||
{ "eagle3", "eagle3-", true, &common_download_opts::download_eagle3, &common_download_hf_plan::eagle3 },
|
||||
{ "dspark", "dspark-", true, &common_download_opts::download_dspark, &common_download_hf_plan::dspark },
|
||||
};
|
||||
|
||||
const std::vector<common_download_sidecar> & common_download_sidecars() {
|
||||
return sidecars;
|
||||
static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files,
|
||||
const std::string & model) {
|
||||
return find_best_sibling(files, model, "mmproj");
|
||||
}
|
||||
|
||||
bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind) {
|
||||
for (const auto & s : sidecars) {
|
||||
if (kind == s.name) {
|
||||
opts.*s.opt = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files,
|
||||
const std::string & model,
|
||||
const std::string & tag = "") {
|
||||
return find_best_sibling(files, model, "mtp-", tag);
|
||||
}
|
||||
|
||||
std::vector<std::string> common_download_plan_sidecars(const common_download_hf_plan & plan) {
|
||||
std::vector<std::string> out;
|
||||
for (const auto & s : sidecars) {
|
||||
if (!(plan.*s.file).path.empty()) {
|
||||
out.emplace_back(s.name);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files,
|
||||
const std::string & model,
|
||||
const std::string & tag = "") {
|
||||
return find_best_sibling(files, model, "eagle3-", tag);
|
||||
}
|
||||
|
||||
std::vector<common_download_partial> common_download_plan_partials(const common_download_hf_plan & plan) {
|
||||
std::vector<common_download_partial> out;
|
||||
|
||||
// a file in the cache resolves to its snapshot entry, a file still to download to its blob
|
||||
auto add = [&](const std::string & kind, const hf_cache::hf_file & f) {
|
||||
if (!f.local_path.empty() && f.local_path != f.final_path) {
|
||||
out.push_back({ kind, f.local_path + PARTIAL_SUFFIX });
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto & f : plan.model_files) {
|
||||
add("", f);
|
||||
}
|
||||
for (const auto & s : sidecars) {
|
||||
add(s.name, plan.*s.file);
|
||||
}
|
||||
return out;
|
||||
static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
|
||||
const std::string & model,
|
||||
const std::string & tag = "") {
|
||||
return find_best_sibling(files, model, "dflash-", tag);
|
||||
}
|
||||
|
||||
// true when `name` carries the keyword of a sidecar kind
|
||||
static bool name_is_sidecar(const std::string & name) {
|
||||
for (const auto & s : sidecars) {
|
||||
if (name.find(s.keyword) != std::string::npos) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files,
|
||||
const std::string & model,
|
||||
const std::string & tag = "") {
|
||||
return find_best_sibling(files, model, "dspark-", tag);
|
||||
}
|
||||
|
||||
static bool gguf_filename_is_model(const std::string & filepath) {
|
||||
@@ -750,7 +633,12 @@ static bool gguf_filename_is_model(const std::string & filepath) {
|
||||
filename = filename.substr(pos + 1);
|
||||
}
|
||||
|
||||
return filename.find("imatrix") == std::string::npos && !name_is_sidecar(filename);
|
||||
return filename.find("mmproj") == std::string::npos &&
|
||||
filename.find("imatrix") == std::string::npos &&
|
||||
filename.find("mtp-") == std::string::npos &&
|
||||
filename.find("eagle3-") == std::string::npos &&
|
||||
filename.find("dflash-") == std::string::npos &&
|
||||
filename.find("dspark-") == std::string::npos;
|
||||
}
|
||||
|
||||
static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
|
||||
@@ -764,8 +652,7 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
|
||||
}
|
||||
|
||||
for (const auto & t : tags) {
|
||||
// match the tag as a full token, so ":F16" does not match a BF16 file
|
||||
std::regex pattern("(^|[-./])" + t + "[.-]", std::regex::icase);
|
||||
std::regex pattern(t + "[.-]", std::regex::icase);
|
||||
for (const auto & f : files) {
|
||||
if (gguf_filename_is_model(f.path) &&
|
||||
std::regex_search(f.path, pattern)) {
|
||||
@@ -843,11 +730,8 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
|
||||
}
|
||||
} else {
|
||||
primary = find_best_model(all, tag);
|
||||
// a requested draft can resolve on its own, without a full model of the same tag
|
||||
const bool want_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) {
|
||||
return s.draft && opts.*s.opt;
|
||||
});
|
||||
if (primary.path.empty() && !want_draft) {
|
||||
// a requested sidecar can resolve on its own, without a full model of the same tag
|
||||
if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) {
|
||||
LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
|
||||
list_available_gguf_files(all);
|
||||
return plan;
|
||||
@@ -859,22 +743,24 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
|
||||
plan.model_files = get_split_files(all, primary);
|
||||
}
|
||||
|
||||
// a draft resolves at the tag, the mmproj pairs with the model
|
||||
for (const auto & s : sidecars) {
|
||||
if (!(opts.*s.opt)) {
|
||||
continue;
|
||||
}
|
||||
if (s.draft) {
|
||||
plan.*s.file = find_best_sibling(all, primary.path, s.keyword, tag);
|
||||
} else if (!primary.path.empty()) {
|
||||
plan.*s.file = find_best_sibling(all, primary.path, s.keyword);
|
||||
}
|
||||
if (opts.download_mmproj && !primary.path.empty()) {
|
||||
plan.mmproj = find_best_mmproj(all, primary.path);
|
||||
}
|
||||
if (opts.download_mtp) {
|
||||
plan.mtp = find_best_mtp(all, primary.path, tag);
|
||||
}
|
||||
if (opts.download_dflash) {
|
||||
plan.dflash = find_best_dflash(all, primary.path, tag);
|
||||
}
|
||||
if (opts.download_eagle3) {
|
||||
plan.eagle3 = find_best_eagle3(all, primary.path, tag);
|
||||
}
|
||||
if (opts.download_dspark) {
|
||||
plan.dspark = find_best_dspark(all, primary.path, tag);
|
||||
}
|
||||
|
||||
const bool has_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) {
|
||||
return s.draft && !(plan.*s.file).local_path.empty();
|
||||
});
|
||||
if (primary.path.empty() && !has_draft) {
|
||||
if (primary.path.empty() &&
|
||||
plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) {
|
||||
LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
|
||||
list_available_gguf_files(all);
|
||||
}
|
||||
@@ -887,7 +773,7 @@ void common_download_run_tasks(const std::vector<common_download_task> & tasks)
|
||||
for (const auto & task : tasks) {
|
||||
futures.push_back(std::async(std::launch::async,
|
||||
[&task]() {
|
||||
return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf, task.lock_path, task.alt_path);
|
||||
return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf);
|
||||
}
|
||||
));
|
||||
}
|
||||
@@ -1022,7 +908,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
|
||||
std::string model_filename = repo;
|
||||
std::replace(model_filename.begin(), model_filename.end(), '/', '_');
|
||||
model_filename += "_" + tag + ".gguf";
|
||||
std::string local_path = fs_get_cache_file(model_filename).string();
|
||||
std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
|
||||
|
||||
const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
|
||||
common_download_opts opts;
|
||||
@@ -1040,11 +926,6 @@ std::string common_docker_resolve_model(const std::string & docker) {
|
||||
}
|
||||
}
|
||||
|
||||
// a cached file that lists a model under its tag: the first split of a model, not a sidecar
|
||||
static bool is_listed_model(const gguf_split_info & split) {
|
||||
return split.index == 1 && !split.tag.empty() && !name_is_sidecar(split.prefix);
|
||||
}
|
||||
|
||||
std::vector<common_cached_model_info> common_list_cached_models() {
|
||||
std::unordered_set<std::string> seen;
|
||||
std::vector<common_cached_model_info> result;
|
||||
@@ -1053,7 +934,12 @@ std::vector<common_cached_model_info> common_list_cached_models() {
|
||||
|
||||
for (const auto & f : files) {
|
||||
auto split = get_gguf_split_info(f.path);
|
||||
if (!is_listed_model(split)) {
|
||||
if (split.index != 1 || split.tag.empty() ||
|
||||
split.prefix.find("mmproj") != std::string::npos ||
|
||||
split.prefix.find("mtp-") != std::string::npos ||
|
||||
split.prefix.find("eagle3-") != std::string::npos ||
|
||||
split.prefix.find("dflash-") != std::string::npos ||
|
||||
split.prefix.find("dspark-") != std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
if (seen.insert(f.repo_id + ":" + split.tag).second) {
|
||||
@@ -1084,44 +970,6 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c
|
||||
return find_best_model(files, tag).local_path;
|
||||
}
|
||||
|
||||
// the files -hf loads from the cache, the model with every sidecar, for each cached
|
||||
// tag of `repo_id` other than `tag`
|
||||
static std::unordered_set<std::string> files_in_use(const std::string & repo_id, const std::string & tag) {
|
||||
common_download_opts opts;
|
||||
opts.offline = true;
|
||||
for (const auto & s : sidecars) {
|
||||
opts.*s.opt = true;
|
||||
}
|
||||
|
||||
std::unordered_set<std::string> tags;
|
||||
std::unordered_set<std::string> used;
|
||||
|
||||
for (const auto & f : hf_cache::get_cached_files(repo_id)) {
|
||||
auto split = get_gguf_split_info(f.path);
|
||||
if (!is_listed_model(split) || split.tag == tag || !tags.insert(split.tag).second) {
|
||||
continue;
|
||||
}
|
||||
|
||||
common_params_model model;
|
||||
model.hf_repo = repo_id + ":" + split.tag;
|
||||
|
||||
auto plan = common_download_get_hf_plan(model, opts);
|
||||
|
||||
for (const auto & m : plan.model_files) {
|
||||
used.insert(m.local_path);
|
||||
}
|
||||
for (const auto & s : sidecars) {
|
||||
const auto & sf = plan.*s.file;
|
||||
if (!sf.local_path.empty()) {
|
||||
used.insert(sf.local_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
return used;
|
||||
}
|
||||
|
||||
static bool remove_snapshot_entries(const std::string & repo_id, const std::vector<std::filesystem::path> & to_remove);
|
||||
|
||||
bool common_download_remove(const std::string & hf_repo_with_tag) {
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
@@ -1141,13 +989,11 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// collect snapshot entries whose tag matches, but a file another tag loads stays
|
||||
const auto used = files_in_use(repo_id, tag_upper);
|
||||
|
||||
// collect snapshot entries whose tag matches
|
||||
std::vector<fs::path> to_remove;
|
||||
for (const auto & f : files) {
|
||||
auto split = get_gguf_split_info(f.path);
|
||||
if (split.tag == tag_upper && used.find(f.local_path) == used.end()) {
|
||||
if (split.tag == tag_upper) {
|
||||
to_remove.emplace_back(f.local_path);
|
||||
}
|
||||
}
|
||||
@@ -1156,13 +1002,6 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return remove_snapshot_entries(repo_id, to_remove);
|
||||
}
|
||||
|
||||
// remove snapshot entries of `repo_id`, then the blobs no remaining entry references
|
||||
static bool remove_snapshot_entries(const std::string & repo_id, const std::vector<std::filesystem::path> & to_remove) {
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
// resolve blob paths from symlinks before deleting snapshot entries
|
||||
std::vector<fs::path> blobs_to_check;
|
||||
for (const auto & p : to_remove) {
|
||||
@@ -1214,26 +1053,3 @@ static bool remove_snapshot_entries(const std::string & repo_id, const std::vect
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts) {
|
||||
common_params_model model;
|
||||
model.hf_repo = hf_repo_with_tag;
|
||||
|
||||
common_download_opts cached = opts;
|
||||
cached.offline = true;
|
||||
|
||||
auto plan = common_download_get_hf_plan(model, cached);
|
||||
|
||||
std::vector<std::filesystem::path> to_remove;
|
||||
for (const auto & s : sidecars) {
|
||||
const auto & f = plan.*s.file;
|
||||
if (!f.local_path.empty()) {
|
||||
to_remove.emplace_back(f.local_path);
|
||||
}
|
||||
}
|
||||
if (to_remove.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return remove_snapshot_entries(common_download_split_repo_tag(hf_repo_with_tag).first, to_remove);
|
||||
}
|
||||
+3
-49
@@ -31,8 +31,6 @@ struct common_remote_params {
|
||||
common_header_list headers;
|
||||
long timeout = 0; // in seconds, 0 means no timeout
|
||||
long max_size = 0; // unlimited if 0
|
||||
// if set: called per hop (redirects too), returns the address to connect to, throws to refuse
|
||||
std::function<std::string(const std::string & host)> resolve_host;
|
||||
};
|
||||
|
||||
// get remote file content, returns <http_code, raw_response_body>
|
||||
@@ -71,15 +69,12 @@ struct common_download_task {
|
||||
std::string local_path;
|
||||
std::function<void()> on_done;
|
||||
bool is_hf = false;
|
||||
std::string lock_path;
|
||||
std::string alt_path; // HF snapshot path, also treated as cached (the blob may be moved there)
|
||||
|
||||
common_download_task() = default;
|
||||
common_download_task(hf_cache::hf_file f,
|
||||
const common_download_opts & opts,
|
||||
std::function<void()> on_done = nullptr)
|
||||
: opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true),
|
||||
lock_path(hf_cache::get_lock_path(f.repo_id, f.oid)), alt_path(f.final_path) {}
|
||||
: opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true) {}
|
||||
};
|
||||
|
||||
void common_download_run_tasks(const std::vector<common_download_task> & tasks);
|
||||
@@ -97,14 +92,10 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c
|
||||
// download single file from url to local path
|
||||
// returns status code or -1 on error
|
||||
// skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash)
|
||||
// lock_path: lock file used to serialize downloads of the same file; when empty, "<path>.lock" is used
|
||||
// alt_path: additional path treated as present in cache (see cache_hit in download.cpp)
|
||||
int common_download_file_single(const std::string & url,
|
||||
const std::string & path,
|
||||
const common_download_opts & opts = {},
|
||||
bool skip_etag = false,
|
||||
const std::string & lock_path = "",
|
||||
const std::string & alt_path = "");
|
||||
bool skip_etag = false);
|
||||
|
||||
// resolve and download model from Docker registry
|
||||
// return local path to downloaded model file
|
||||
@@ -113,17 +104,10 @@ std::string common_docker_resolve_model(const std::string & docker);
|
||||
// Remove a cached model from disk
|
||||
// input format: "user/model" or "user/model:tag"
|
||||
// - if tag is omitted, removes the entire repo cache directory
|
||||
// - if tag is present, removes only files matching that tag (and orphaned blobs),
|
||||
// except the files another cached tag of the repo still loads
|
||||
// - if tag is present, removes only files matching that tag (and orphaned blobs)
|
||||
// returns true if anything was removed
|
||||
bool common_download_remove(const std::string & hf_repo_with_tag);
|
||||
|
||||
// remove the cached sidecar files `opts` selects for "user/model:tag", as the cached
|
||||
// plan resolves them, and leave the model files in place; a sidecar file another tag
|
||||
// shares goes for every tag that resolves it
|
||||
// returns true if anything was removed
|
||||
bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts);
|
||||
|
||||
struct common_download_hf_plan {
|
||||
hf_cache::hf_file primary;
|
||||
hf_cache::hf_files model_files;
|
||||
@@ -135,33 +119,3 @@ struct common_download_hf_plan {
|
||||
hf_cache::hf_file preset; // if set, only this file is downloaded
|
||||
};
|
||||
common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);
|
||||
|
||||
// a sidecar kind a download can pull next to its model: mmproj, and the draft heads
|
||||
// of --spec-type draft-<name>
|
||||
struct common_download_sidecar {
|
||||
const char * name; // the kind, as the API names it
|
||||
const char * keyword; // the part of a file name that marks a file of this kind
|
||||
bool draft; // a draft head, resolved at the tag even without a model
|
||||
|
||||
bool common_download_opts::* opt; // the option that requests it
|
||||
hf_cache::hf_file common_download_hf_plan::* file; // the plan file it resolves to
|
||||
};
|
||||
|
||||
// every sidecar kind, the single list the resolution, the listing and the removal follow
|
||||
const std::vector<common_download_sidecar> & common_download_sidecars();
|
||||
|
||||
// select the sidecar `kind` in `opts`; returns false for an unknown kind
|
||||
bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind);
|
||||
|
||||
// the sidecar kinds `plan` resolves a file for
|
||||
std::vector<std::string> common_download_plan_sidecars(const common_download_hf_plan & plan);
|
||||
|
||||
// a file `plan` still has to download, at the path its partial takes while it is
|
||||
// incomplete; kind is the sidecar it belongs to, empty for the model files
|
||||
struct common_download_partial {
|
||||
std::string kind;
|
||||
std::string path;
|
||||
};
|
||||
|
||||
// the partials the download of `plan` writes, for the files not in the cache yet
|
||||
std::vector<common_download_partial> common_download_plan_partials(const common_download_hf_plan & plan);
|
||||
@@ -1,179 +0,0 @@
|
||||
#include "file-lock.h"
|
||||
|
||||
#include "log.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <filesystem>
|
||||
#include <thread>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#include <io.h>
|
||||
#else
|
||||
#include <sys/file.h>
|
||||
#include <unistd.h>
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#endif
|
||||
|
||||
common_file_lock::common_file_lock(const std::string & lock_path) {
|
||||
this->lock_path = lock_path;
|
||||
}
|
||||
|
||||
#if defined(_WIN32)
|
||||
// the \\?\ prefix (for paths longer than 255 chars) is only accepted on absolute paths
|
||||
// with backslash separators: normalize first, then check the length of the absolute form
|
||||
static std::wstring as_extended_path(const std::string & path) {
|
||||
// convert from the narrow encoding std::filesystem uses on this system
|
||||
int wlen = MultiByteToWideChar(CP_ACP, 0, path.c_str(), (int) path.size(), nullptr, 0);
|
||||
if (wlen <= 0) {
|
||||
return std::wstring(path.begin(), path.end()); // best effort for exotic encodings
|
||||
}
|
||||
std::wstring wide(wlen, L'\0');
|
||||
MultiByteToWideChar(CP_ACP, 0, path.c_str(), (int) path.size(), wide.data(), wlen);
|
||||
|
||||
if (wide.rfind(L"\\\\?\\", 0) == 0) {
|
||||
return wide;
|
||||
}
|
||||
|
||||
// GetFullPathNameW resolves relative paths and normalizes separators
|
||||
DWORD size = GetFullPathNameW(wide.c_str(), 0, nullptr, nullptr);
|
||||
if (size == 0) {
|
||||
return wide;
|
||||
}
|
||||
std::wstring full(size, L'\0');
|
||||
DWORD written = GetFullPathNameW(wide.c_str(), size, full.data(), nullptr);
|
||||
if (written == 0 || written >= size) {
|
||||
return wide;
|
||||
}
|
||||
full.resize(written);
|
||||
|
||||
if (full.size() <= 255) {
|
||||
return wide; // short enough: open the path exactly as given
|
||||
}
|
||||
if (full.rfind(L"\\\\", 0) == 0) {
|
||||
return L"\\\\?\\UNC\\" + full.substr(2); // \\server\share -> \\?\UNC\server\share
|
||||
}
|
||||
return L"\\\\?\\" + full;
|
||||
}
|
||||
#endif
|
||||
|
||||
common_file_lock::status common_file_lock::try_lock() {
|
||||
if (lock_path.empty()) {
|
||||
return status::acquired; // no lock path, nothing to lock
|
||||
}
|
||||
#if defined(_WIN32)
|
||||
if (fd < 0) {
|
||||
// opened with a wide path: extended paths cannot go through the narrow CRT
|
||||
std::wstring lock_path_w = as_extended_path(lock_path);
|
||||
fd = _wopen(lock_path_w.c_str(), _O_RDWR | _O_CREAT | _O_BINARY | _O_NOINHERIT, _S_IREAD | _S_IWRITE);
|
||||
if (fd < 0) {
|
||||
LOG_ERR("%s: cannot open lock file: %s\n", __func__, lock_path.c_str());
|
||||
return status::error;
|
||||
}
|
||||
}
|
||||
// lock the first byte, the file itself stays empty
|
||||
OVERLAPPED ov = {};
|
||||
locked = LockFileEx((HANDLE) _get_osfhandle(fd),
|
||||
LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &ov);
|
||||
if (locked) {
|
||||
return status::acquired;
|
||||
}
|
||||
if (GetLastError() == ERROR_LOCK_VIOLATION) {
|
||||
return status::busy;
|
||||
}
|
||||
LOG_ERR("%s: file locking failed (%d)\n", __func__, (int) GetLastError());
|
||||
return status::error;
|
||||
#else
|
||||
if (fd < 0) {
|
||||
int open_flags = O_RDWR | O_CREAT | O_CLOEXEC;
|
||||
#ifdef O_NOFOLLOW
|
||||
open_flags |= O_NOFOLLOW;
|
||||
#endif
|
||||
// group-writable, so users sharing a cache can wait on the lock
|
||||
fd = open(lock_path.c_str(), open_flags, 0664);
|
||||
if (fd < 0) {
|
||||
LOG_ERR("%s: cannot open lock file: %s\n", __func__, lock_path.c_str());
|
||||
return status::error;
|
||||
}
|
||||
// open() filters the mode through the umask
|
||||
(void) fchmod(fd, 0664);
|
||||
}
|
||||
|
||||
if (flock(fd, LOCK_EX | LOCK_NB) == 0) {
|
||||
locked = true;
|
||||
return status::acquired;
|
||||
}
|
||||
// EACCES is reported as contention on some filesystems
|
||||
if (errno == EWOULDBLOCK || errno == EAGAIN || errno == EACCES) {
|
||||
return status::busy;
|
||||
}
|
||||
// flock is not implemented (some NFS/FUSE mounts): report it instead of racing
|
||||
LOG_ERR("%s: file locking failed (%s)\n", __func__, strerror(errno));
|
||||
return status::error;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool common_file_lock::acquire(const std::function<bool()> & keep_waiting) {
|
||||
// the lock file is opened by try_lock, its parent must exist by then
|
||||
if (!lock_path.empty()) {
|
||||
std::error_code ec;
|
||||
auto lock_dir = std::filesystem::path(lock_path).parent_path();
|
||||
if (!lock_dir.empty()) {
|
||||
std::filesystem::create_directories(lock_dir, ec);
|
||||
if (ec) {
|
||||
LOG_ERR("%s: unable to create lock directory '%s': %s\n", __func__, lock_dir.string().c_str(), ec.message().c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
switch (try_lock()) {
|
||||
case status::acquired:
|
||||
return true;
|
||||
case status::busy:
|
||||
if (keep_waiting && !keep_waiting()) {
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
continue;
|
||||
case status::error:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// the lock file is never deleted: another process could then create a fresh
|
||||
// file at the same path and lock that
|
||||
void common_file_lock::close() {
|
||||
#if defined(_WIN32)
|
||||
if (fd >= 0) {
|
||||
if (locked) {
|
||||
OVERLAPPED ov = {};
|
||||
UnlockFileEx((HANDLE) _get_osfhandle(fd), 0, 1, 0, &ov);
|
||||
}
|
||||
_close(fd);
|
||||
}
|
||||
#else
|
||||
if (fd >= 0) {
|
||||
if (locked) {
|
||||
flock(fd, LOCK_UN);
|
||||
}
|
||||
::close(fd);
|
||||
}
|
||||
#endif
|
||||
fd = -1;
|
||||
locked = false;
|
||||
}
|
||||
|
||||
common_file_lock::~common_file_lock() {
|
||||
close();
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
// Cross-process exclusive lock (flock on POSIX, LockFileEx on Windows).
|
||||
// Lazy: acquire() opens the lock file, so a read-only cache works if the lock is never acquired.
|
||||
class common_file_lock {
|
||||
public:
|
||||
common_file_lock(const std::string & lock_path);
|
||||
~common_file_lock();
|
||||
|
||||
// take the lock, waiting while another process holds it;
|
||||
// keep_waiting is polled before each retry, false cancels the wait.
|
||||
// Also returns false when locking is unavailable.
|
||||
bool acquire(const std::function<bool()> & keep_waiting = {});
|
||||
|
||||
void close();
|
||||
|
||||
common_file_lock(const common_file_lock &) = delete;
|
||||
common_file_lock & operator=(const common_file_lock &) = delete;
|
||||
|
||||
private:
|
||||
enum class status {
|
||||
acquired,
|
||||
busy, // another process holds the lock
|
||||
error, // file locking is unavailable: no exclusion is possible
|
||||
};
|
||||
|
||||
status try_lock();
|
||||
|
||||
std::string lock_path;
|
||||
|
||||
int fd = -1;
|
||||
bool locked = false;
|
||||
};
|
||||
+118
-233
@@ -1,8 +1,6 @@
|
||||
#include "fit.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "json.h"
|
||||
#include "llama-cpp.h"
|
||||
#include "log.h"
|
||||
|
||||
#include "../src/llama-ext.h"
|
||||
@@ -29,56 +27,46 @@ class common_params_fit_exception : public std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
// RAII: send llama logs below min_level to the debug log
|
||||
struct common_fit_log_scope {
|
||||
common_fit_log_scope(ggml_log_level min_level) : min_level(min_level) {
|
||||
llama_log_get(&original_logger.callback, &original_logger.user_data);
|
||||
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
|
||||
const common_fit_log_scope * self = (const common_fit_log_scope *) user_data;
|
||||
const ggml_log_level level_eff = level >= self->min_level ? level : GGML_LOG_LEVEL_DEBUG;
|
||||
self->original_logger.callback(level_eff, text, self->original_logger.user_data);
|
||||
}, this);
|
||||
}
|
||||
|
||||
~common_fit_log_scope() {
|
||||
llama_log_set(original_logger.callback, original_logger.user_data);
|
||||
}
|
||||
|
||||
common_fit_log_scope(const common_fit_log_scope &) = delete;
|
||||
common_fit_log_scope & operator=(const common_fit_log_scope &) = delete;
|
||||
|
||||
private:
|
||||
struct {
|
||||
ggml_log_callback callback;
|
||||
void * user_data;
|
||||
} original_logger;
|
||||
ggml_log_level min_level;
|
||||
};
|
||||
|
||||
static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
common_fit_log_scope log_scope(log_level);
|
||||
struct user_data_t {
|
||||
struct {
|
||||
ggml_log_callback callback;
|
||||
void * user_data;
|
||||
} original_logger;
|
||||
ggml_log_level min_level; // prints below this log level go to debug log
|
||||
};
|
||||
user_data_t ud;
|
||||
llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
|
||||
ud.min_level = log_level;
|
||||
|
||||
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
|
||||
const user_data_t * ud = (const user_data_t *) user_data;
|
||||
const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
|
||||
ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
|
||||
}, &ud);
|
||||
|
||||
llama_model_params mparams_copy = *mparams;
|
||||
mparams_copy.no_alloc = true;
|
||||
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model * model = common_model_load(paths_model, mparams_copy);
|
||||
llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
|
||||
if (model == nullptr) {
|
||||
llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
|
||||
throw std::runtime_error("failed to load model");
|
||||
}
|
||||
|
||||
llama_context * ctx = llama_init_from_model(model, *cparams);
|
||||
if (ctx == nullptr) {
|
||||
llama_model_free(model);
|
||||
llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
|
||||
throw std::runtime_error("failed to create llama_context from model");
|
||||
}
|
||||
|
||||
@@ -153,8 +141,6 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
if (mparams->load_mtp) {
|
||||
hp_ngl += llama_model_n_layer_nextn(model);
|
||||
}
|
||||
// without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers
|
||||
hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model);
|
||||
hp_n_ctx_train = llama_model_n_ctx_train(model);
|
||||
hp_n_expert = llama_model_n_expert(model);
|
||||
|
||||
@@ -162,93 +148,22 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
||||
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> common_fit_measure(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
uint32_t * n_ctx_train,
|
||||
uint32_t * n_ctx,
|
||||
ggml_log_level log_level) {
|
||||
common_fit_log_scope log_scope(log_level);
|
||||
|
||||
llama_model_params mparams_copy = *mparams;
|
||||
mparams_copy.no_alloc = true;
|
||||
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model_ptr model(common_model_load(paths_model, mparams_copy));
|
||||
if (!model) {
|
||||
throw std::runtime_error("failed to load model");
|
||||
}
|
||||
|
||||
llama_context_ptr ctx(llama_init_from_model(model.get(), *cparams));
|
||||
if (!ctx) {
|
||||
throw std::runtime_error("failed to create llama_context from model");
|
||||
}
|
||||
|
||||
if (n_ctx_train) {
|
||||
*n_ctx_train = llama_model_n_ctx_train(model.get());
|
||||
}
|
||||
if (n_ctx) {
|
||||
*n_ctx = llama_n_ctx(ctx.get());
|
||||
}
|
||||
|
||||
return common_memory_breakdown_rows(ctx.get());
|
||||
}
|
||||
|
||||
common_memory_reports common_measure_memory(common_params & params) {
|
||||
const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
|
||||
const std::vector<std::string> paths_model = params.model.get_paths();
|
||||
|
||||
common_memory_reports res;
|
||||
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
||||
// max: what --fit picks now, no-op with -fit off
|
||||
{
|
||||
auto mparams_fit = mparams;
|
||||
auto cparams_fit = cparams;
|
||||
common_fit_apply(params, mparams_fit, cparams_fit);
|
||||
res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
|
||||
}
|
||||
|
||||
// min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
|
||||
uint32_t n_ctx_min = cparams.n_ctx;
|
||||
if (n_ctx_min == 0 && params.fit_params) {
|
||||
const uint32_t n_streams = cparams.kv_unified ? 1 : std::max<uint32_t>(1, cparams.n_seq_max);
|
||||
// -c 0 sets fit_params_min_ctx to UINT32_MAX, cap at n_ctx_train like --fit
|
||||
n_ctx_min = (uint32_t) std::min<uint64_t>(
|
||||
uint64_t(params.fit_params_min_ctx) * n_streams,
|
||||
uint64_t(res.n_ctx_train) * n_streams);
|
||||
}
|
||||
if (n_ctx_min == 0 || n_ctx_min >= res.n_ctx_max) {
|
||||
// nothing to reduce, or -fit off with the trained context
|
||||
res.n_ctx_min = res.n_ctx_max;
|
||||
res.info_min = res.info_max;
|
||||
} else {
|
||||
cparams.n_ctx = n_ctx_min;
|
||||
res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
common_device_memory_data_vec common_get_device_memory_data(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
const llama_model_params * mparams,
|
||||
const llama_context_params * cparams,
|
||||
std::vector<ggml_backend_dev_t> & devs,
|
||||
uint32_t & hp_ngl,
|
||||
uint32_t & hp_ngl_skip,
|
||||
uint32_t & hp_n_ctx_train,
|
||||
uint32_t & hp_n_expert,
|
||||
ggml_log_level log_level) {
|
||||
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
|
||||
|
||||
common_device_memory_data_vec ret(impl.size());
|
||||
for (size_t i = 0; i < impl.size(); i++) {
|
||||
@@ -262,7 +177,7 @@ common_device_memory_data_vec common_get_device_memory_data(
|
||||
}
|
||||
|
||||
static void common_params_fit_impl(
|
||||
const std::vector<std::string> & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
|
||||
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
|
||||
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
|
||||
size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {
|
||||
if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {
|
||||
@@ -274,7 +189,6 @@ static void common_params_fit_impl(
|
||||
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
@@ -296,7 +210,6 @@ static void common_params_fit_impl(
|
||||
if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {
|
||||
std::vector<ggml_backend_dev_t> devs_extra;
|
||||
uint32_t ngl_extra = 0;
|
||||
uint32_t nsk_extra = 0;
|
||||
uint32_t nct_extra = 0;
|
||||
uint32_t nex_extra = 0;
|
||||
|
||||
@@ -308,7 +221,7 @@ static void common_params_fit_impl(
|
||||
dmds_t measured;
|
||||
try {
|
||||
measured = common_get_device_memory_data_impl(
|
||||
extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
|
||||
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
|
||||
} catch (const std::runtime_error & e) {
|
||||
// the extra model is optional, fit the main model alone rather than giving up
|
||||
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
|
||||
@@ -348,7 +261,7 @@ static void common_params_fit_impl(
|
||||
// step 1: get data for default parameters and check whether any changes are necessary in the first place
|
||||
|
||||
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
|
||||
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
|
||||
@@ -360,7 +273,7 @@ static void common_params_fit_impl(
|
||||
if (n_streams > 1) {
|
||||
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
|
||||
__func__, n_ctx_max, n_streams);
|
||||
dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
}
|
||||
}
|
||||
add_extra_memory(dmds_full);
|
||||
@@ -499,7 +412,7 @@ static void common_params_fit_impl(
|
||||
|
||||
int64_t sum_projected_used_min_ctx = 0;
|
||||
cparams->n_ctx = n_ctx_min_total;
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_min_ctx);
|
||||
if (nd == 0) {
|
||||
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
|
||||
@@ -646,20 +559,6 @@ static void common_params_fit_impl(
|
||||
assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);
|
||||
uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides
|
||||
|
||||
// the skipped NextN layers come right before the output layer, put them on the same device as the output layer
|
||||
// so that the layers assigned by the model loader match the layers above
|
||||
if (hp_nsk > 0 && mparams.n_gpu_layers > 0) {
|
||||
mparams.n_gpu_layers += hp_nsk;
|
||||
if (nd > 1) {
|
||||
// the last devices have no layers if not even the output layer fits
|
||||
size_t id_last = nd - 1;
|
||||
while (ngl_per_device[id_last].n_layer == 0) {
|
||||
id_last--;
|
||||
}
|
||||
tensor_split[id_last] += hp_nsk;
|
||||
}
|
||||
}
|
||||
|
||||
mparams.tensor_split = tensor_split;
|
||||
|
||||
size_t itbo = 0;
|
||||
@@ -695,7 +594,7 @@ static void common_params_fit_impl(
|
||||
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
|
||||
|
||||
dmds_t dmd_nl = common_get_device_memory_data_impl(
|
||||
paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmd_nl);
|
||||
|
||||
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
|
||||
@@ -724,7 +623,7 @@ static void common_params_fit_impl(
|
||||
|
||||
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
|
||||
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
|
||||
paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
|
||||
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
add_extra_memory(dmds_cpu_moe);
|
||||
|
||||
for (size_t id = 0; id < nd; id++) {
|
||||
@@ -978,7 +877,7 @@ static void common_params_fit_impl(
|
||||
}
|
||||
|
||||
enum common_params_fit_status common_fit_params(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams,
|
||||
float * tensor_split,
|
||||
@@ -990,7 +889,7 @@ enum common_params_fit_status common_fit_params(
|
||||
const int64_t t0_us = llama_time_us();
|
||||
common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
|
||||
try {
|
||||
common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
|
||||
common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
|
||||
LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
|
||||
} catch (const common_params_fit_exception & e) {
|
||||
LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
|
||||
@@ -1004,7 +903,8 @@ enum common_params_fit_status common_fit_params(
|
||||
return status;
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const struct llama_context * ctx) {
|
||||
void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
//const auto & devices = ctx->get_model().devices;
|
||||
const auto * model = llama_get_model(ctx);
|
||||
|
||||
std::vector<ggml_backend_dev_t> devices;
|
||||
@@ -1014,6 +914,18 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
|
||||
|
||||
llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);
|
||||
|
||||
std::vector<std::array<std::string, 9>> table_data;
|
||||
table_data.reserve(devices.size());
|
||||
|
||||
// same data as the table below, for --log-jsonl consumers
|
||||
common_json rows = common_json::array();
|
||||
const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
|
||||
const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
|
||||
const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
|
||||
|
||||
table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
|
||||
|
||||
constexpr size_t MiB = 1024 * 1024;
|
||||
const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};
|
||||
|
||||
// track seen buffer types to avoid double counting:
|
||||
@@ -1052,130 +964,104 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<common_memory_breakdown_row> rows;
|
||||
rows.reserve(devices.size() + 1 + memory_breakdown.size());
|
||||
|
||||
// print memory breakdown for each device:
|
||||
for (size_t i = 0; i < devices.size(); i++) {
|
||||
ggml_backend_dev_t dev = devices[i];
|
||||
llama_memory_breakdown_data mb = mb_dev[i];
|
||||
|
||||
common_memory_breakdown_row row;
|
||||
row.kind = common_memory_breakdown_row::DEVICE;
|
||||
row.name = ggml_backend_dev_name(dev);
|
||||
row.description = ggml_backend_dev_description(dev);
|
||||
const std::string name = ggml_backend_dev_name(dev);
|
||||
std::string desc = ggml_backend_dev_description(dev);
|
||||
for (const std::string & prefix : desc_prefixes_strip) {
|
||||
if (row.description.length() >= prefix.length() && row.description.substr(0, prefix.length()) == prefix) {
|
||||
row.description = row.description.substr(prefix.length());
|
||||
if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {
|
||||
desc = desc.substr(prefix.length());
|
||||
}
|
||||
}
|
||||
|
||||
size_t free, total;
|
||||
ggml_backend_dev_memory(dev, &free, &total);
|
||||
row.total = total;
|
||||
row.free = free;
|
||||
row.model = mb_dev[i].model;
|
||||
row.context = mb_dev[i].context;
|
||||
row.compute = mb_dev[i].compute;
|
||||
rows.push_back(std::move(row));
|
||||
|
||||
const size_t self = mb.model + mb.context + mb.compute;
|
||||
const int64_t unaccounted = static_cast<int64_t>(total) - static_cast<int64_t>(free) - static_cast<int64_t>(self);
|
||||
|
||||
table_data.push_back({
|
||||
template_gpu,
|
||||
" - " + name + " (" + desc + ")",
|
||||
std::to_string(total / MiB),
|
||||
std::to_string(free / MiB),
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(mb.model / MiB),
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
std::to_string(unaccounted / static_cast<int64_t>(MiB))});
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "device"},
|
||||
{"name", name},
|
||||
{"description", desc},
|
||||
{"total", total / MiB},
|
||||
{"free", free / MiB},
|
||||
{"self", self / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
{"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
|
||||
});
|
||||
}
|
||||
|
||||
// print memory breakdown for host:
|
||||
{
|
||||
common_memory_breakdown_row row;
|
||||
row.kind = common_memory_breakdown_row::HOST;
|
||||
row.name = "Host";
|
||||
row.model = mb_host.model;
|
||||
row.context = mb_host.context;
|
||||
row.compute = mb_host.compute;
|
||||
rows.push_back(std::move(row));
|
||||
const size_t self = mb_host.model + mb_host.context + mb_host.compute;
|
||||
table_data.push_back({
|
||||
template_other,
|
||||
" - Host",
|
||||
"", // total
|
||||
"", // free
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(mb_host.model / MiB),
|
||||
std::to_string(mb_host.context / MiB),
|
||||
std::to_string(mb_host.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "host"},
|
||||
{"name", "Host"},
|
||||
{"self", self / MiB},
|
||||
{"model", mb_host.model / MiB},
|
||||
{"context", mb_host.context / MiB},
|
||||
{"compute", mb_host.compute / MiB},
|
||||
});
|
||||
}
|
||||
|
||||
// all remaining buffer types:
|
||||
// print memory breakdown for all remaining buffer types:
|
||||
for (const auto & buft_mb : memory_breakdown) {
|
||||
ggml_backend_buffer_type_t buft = buft_mb.first;
|
||||
const llama_memory_breakdown_data & mb = buft_mb.second;
|
||||
if (seen_buffer_types.count(buft) == 1) {
|
||||
continue;
|
||||
}
|
||||
common_memory_breakdown_row row;
|
||||
row.kind = common_memory_breakdown_row::BUFFER_TYPE;
|
||||
row.name = ggml_backend_buft_name(buft);
|
||||
row.model = mb.model;
|
||||
row.context = mb.context;
|
||||
row.compute = mb.compute;
|
||||
rows.push_back(std::move(row));
|
||||
|
||||
seen_buffer_types.insert(buft);
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
const std::vector<common_memory_breakdown_row> rows = common_memory_breakdown_rows(ctx);
|
||||
|
||||
std::vector<std::array<std::string, 9>> table_data;
|
||||
table_data.reserve(rows.size() + 1);
|
||||
|
||||
// same data as the table below, for --log-jsonl consumers
|
||||
common_json json_rows = common_json::array();
|
||||
const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
|
||||
const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
|
||||
const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
|
||||
|
||||
table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
|
||||
|
||||
constexpr size_t MiB = 1024 * 1024;
|
||||
|
||||
for (const auto & row : rows) {
|
||||
const size_t self = row.model + row.context + row.compute;
|
||||
|
||||
if (row.kind == common_memory_breakdown_row::DEVICE) {
|
||||
const int64_t unaccounted = row.total - row.free - static_cast<int64_t>(self);
|
||||
|
||||
table_data.push_back({
|
||||
template_gpu,
|
||||
" - " + row.name + " (" + row.description + ")",
|
||||
std::to_string(row.total / MiB),
|
||||
std::to_string(row.free / MiB),
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(row.model / MiB),
|
||||
std::to_string(row.context / MiB),
|
||||
std::to_string(row.compute / MiB),
|
||||
std::to_string(unaccounted / static_cast<int64_t>(MiB))});
|
||||
|
||||
json_rows.push_back({
|
||||
{"kind", "device"},
|
||||
{"name", row.name},
|
||||
{"description", row.description},
|
||||
{"total", row.total / MiB},
|
||||
{"free", row.free / MiB},
|
||||
{"self", self / MiB},
|
||||
{"model", row.model / MiB},
|
||||
{"context", row.context / MiB},
|
||||
{"compute", row.compute / MiB},
|
||||
{"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string name = ggml_backend_buft_name(buft);
|
||||
const size_t self = mb.model + mb.context + mb.compute;
|
||||
table_data.push_back({
|
||||
template_other,
|
||||
" - " + row.name,
|
||||
" - " + name,
|
||||
"", // total
|
||||
"", // free
|
||||
std::to_string(self / MiB),
|
||||
std::to_string(row.model / MiB),
|
||||
std::to_string(row.context / MiB),
|
||||
std::to_string(row.compute / MiB),
|
||||
std::to_string(mb.model / MiB),
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
json_rows.push_back({
|
||||
{"kind", row.kind == common_memory_breakdown_row::HOST ? "host" : "buffer_type"},
|
||||
{"name", row.name},
|
||||
rows.push_back({
|
||||
{"kind", "buffer_type"},
|
||||
{"name", name},
|
||||
{"self", self / MiB},
|
||||
{"model", row.model / MiB},
|
||||
{"context", row.context / MiB},
|
||||
{"compute", row.compute / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
});
|
||||
|
||||
seen_buffer_types.insert(buft);
|
||||
}
|
||||
|
||||
for (size_t j = 1; j < table_data[0].size(); j++) {
|
||||
@@ -1195,21 +1081,20 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
|
||||
LOG_JSON("fit_memory_breakdown", common_json({
|
||||
{"unit", "MiB"},
|
||||
{"rows", json_rows},
|
||||
{"rows", rows},
|
||||
}));
|
||||
}
|
||||
|
||||
void common_fit_print(
|
||||
const std::vector<std::string> & paths_model,
|
||||
const char * path_model,
|
||||
llama_model_params * mparams,
|
||||
llama_context_params * cparams) {
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
|
||||
uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
|
||||
GGML_ASSERT(dmd.size() == devs.size() + 1);
|
||||
|
||||
for (size_t id = 0; id < devs.size(); id++) {
|
||||
|
||||
Loaded 100 of 3103 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user