Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CUDA: refactor and deduplicate vector FA kernels (#16208)

Browse Source 
* CUDA: refactor and deduplicate vector FA kernels
This commit is contained in:
Johannes Gäßler
2025-09-27 18:45:07 +02:00
committed by GitHub
parent 0499b29c6f
commit 75a3a6c2cd
129 changed files with 1396 additions and 1989 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+9 -17
View File
@@ -3,13 +3,15 @@
from glob import glob
import os
TYPES_KV = ["GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_F16"]
TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0"]
SOURCE_FATTN_VEC = """// This file has been autogenerated by generate_cu_files.py, do not edit manually.
#include "../fattn-vec-f{vkq_size}.cuh"
#include "../fattn-vec.cuh"
DECL_FATTN_VEC_F{vkq_size}_CASE({head_size}, {type_k}, {type_v});
DECL_FATTN_VEC_CASE( 64, {type_k}, {type_v});
DECL_FATTN_VEC_CASE(128, {type_k}, {type_v});
DECL_FATTN_VEC_CASE(256, {type_k}, {type_v});
"""
SOURCE_FATTN_MMA_START = """// This file has been autogenerated by generate_cu_files.py, do not edit manually.
@@ -46,23 +48,13 @@ def get_short_name(long_quant_name):
return long_quant_name.replace("GGML_TYPE_", "").lower()
def get_head_sizes(type_k, type_v):
if type_k == "GGML_TYPE_F16" and type_v == "GGML_TYPE_F16":
return [64, 128, 256]
if type_k == "GGML_TYPE_F16":
return [64, 128]
return [128]
for filename in glob("*.cu"):
os.remove(filename)
for vkq_size in [16, 32]:
for type_k in TYPES_KV:
for type_v in TYPES_KV:
for head_size in get_head_sizes(type_k, type_v):
with open(f"fattn-vec-f{vkq_size}-instance-hs{head_size}-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f:
f.write(SOURCE_FATTN_VEC.format(vkq_size=vkq_size, head_size=head_size, type_k=type_k, type_v=type_v))
for type_k in TYPES_KV:
for type_v in TYPES_KV:
with open(f"fattn-vec-instance-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f:
f.write(SOURCE_FATTN_VEC.format(type_k=type_k, type_v=type_v))
for ncols in [8, 16, 32, 64]:
for ncols2 in [1, 2, 4, 8, 16]: