gguf-py : fix tensor offset overflow in GGUFReader (#32495)
* gguf-py : fix tensor offset overflow in GGUFReader * gguf-py : cast alignment padding to int
This commit is contained in:
1 parent
ffc87c3a79
commit
b069e526de
2 files changed
+14
-4
No files matched your search
@@ -189,7 +189,7 @@ class GGUFReader:
|
||||
raise ValueError('Invalid alignment: must be a non-zero power of two')
|
||||
padding = offs % self.alignment
|
||||
if padding != 0:
|
||||
offs += self.alignment - padding
|
||||
offs += int(self.alignment - padding)
|
||||
self.data_offset = offs
|
||||
self._build_tensors(offs, tensors_fields)
|
||||
|
||||
@@ -350,7 +350,9 @@ class GGUFReader:
|
||||
np_dims = tuple(reversed(dims.tolist()))
|
||||
block_size, type_size = GGML_QUANT_SIZES[ggml_type]
|
||||
n_bytes = n_elems * type_size // block_size
|
||||
data_offs = int(start_offs + offset_tensor[0])
|
||||
data_offs = start_offs + int(offset_tensor[0])
|
||||
if data_offs + n_bytes > self.data.nbytes:
|
||||
raise ValueError(f'Tensor {tensor_name} data exceeds file size {self.data.nbytes}')
|
||||
item_type: npt.DTypeLike
|
||||
if ggml_type == GGMLQuantizationType.F16:
|
||||
item_count = n_elems
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
from gguf.gguf_reader import GGUFReader
|
||||
|
||||
|
||||
def _write_gguf(path, n_dims_field, dims):
|
||||
def _write_gguf(path, n_dims_field, dims, offset=0):
|
||||
buf = b'GGUF' + struct.pack('<IQQ', 3, 1, 0) # version 3, 1 tensor, 0 kv
|
||||
name = b'bad_tensor'
|
||||
buf += struct.pack('<Q', len(name)) + name
|
||||
@@ -13,7 +13,7 @@ def _write_gguf(path, n_dims_field, dims):
|
||||
for d in dims:
|
||||
buf += struct.pack('<Q', d)
|
||||
buf += struct.pack('<I', 0) # dtype F32
|
||||
buf += struct.pack('<Q', 0) # tensor offset
|
||||
buf += struct.pack('<Q', offset) # tensor offset
|
||||
buf += b'\x00' * 64
|
||||
path.write_bytes(buf)
|
||||
|
||||
@@ -35,3 +35,11 @@ def test_dims_product_no_uint64_wraparound(tmp_path):
|
||||
_write_gguf(p, len(dims), dims)
|
||||
with pytest.raises(ValueError):
|
||||
GGUFReader(p)
|
||||
|
||||
|
||||
def test_tensor_offset_no_uint64_wraparound(tmp_path):
|
||||
# offset 2**64 - 1 must be rejected, not wrap around
|
||||
p = tmp_path / 'evil_offset.gguf'
|
||||
_write_gguf(p, 1, [4], offset=2**64 - 1)
|
||||
with pytest.raises(ValueError, match='exceeds file size'):
|
||||
GGUFReader(p)
|
||||
Reference in new issue
Block a user