GGUF Format › Working with GGUF Files
Language AI HandbookAll chapters GGUF Format › Working with GGUF FilesOne section · Sections in this chapter6 / 14 ↓
Reading tools Definitions & inline links
Choose your expertise level to adjust how many terms are explained. Beginners see more tooltips, experts see fewer to maintain reading flow. Hover over underlined terms for instant definitions.
Article links
Make inline references clickable
Let's explore GGUF files programmatically. While llama.cpp's command-line tools are useful for quick conversions, Python libraries provide a more interactive and scriptable way to inspect and manipulate these files. Building your own parser, even a simple one, is one of the best ways to develop a concrete understanding of how the format works.
The GGUF binary format uses little-endian byte order throughout. All multi-byte integers are stored with the least significant byte first, which matches the native byte order of x86 and most ARM processors. Strings are length-prefixed with a 64-bit unsigned integer giving the byte length, followed by the raw UTF-8 bytes. There are no null terminators. Arrays are type-prefixed: a 32-bit type code followed by a 64-bit element count, followed by that many elements of the specified type.
# GGUF constants
GGUF_MAGIC = 0x46554747 # "GGUF" in little-endian
GGUF_VERSION = 3
# Value type codes
GGUF_TYPE_UINT8 = 0
GGUF_TYPE_INT8 = 1
GGUF_TYPE_UINT16 = 2
GGUF_TYPE_INT16 = 3
GGUF_TYPE_UINT32 = 4
GGUF_TYPE_INT32 = 5
GGUF_TYPE_FLOAT32 = 6
GGUF_TYPE_BOOL = 7
GGUF_TYPE_STRING = 8
GGUF_TYPE_ARRAY = 9
GGUF_TYPE_UINT64 = 10
GGUF_TYPE_INT64 = 11
GGUF_TYPE_FLOAT64 = 12
TYPE_NAMES = {
0: "uint8",
1: "int8",
2: "uint16",
3: "int16",
4: "uint32",
5: "int32",
6: "float32",
7: "bool",
8: "string",
9: "array",
10: "uint64",
11: "int64",
12: "float64",
}import struct
def read_string(f):
"""Read a GGUF string (length-prefixed UTF-8)."""
length = struct.unpack("<Q", f.read(8))[0]
return f.read(length).decode("utf-8")
def read_value(f, value_type):
"""Read a value of the given type."""
if value_type == GGUF_TYPE_UINT8:
return struct.unpack("<B", f.read(1))[0]
elif value_type == GGUF_TYPE_INT8:
return struct.unpack("<b", f.read(1))[0]
elif value_type == GGUF_TYPE_UINT16:
return struct.unpack("<H", f.read(2))[0]
elif value_type == GGUF_TYPE_INT16:
return struct.unpack("<h", f.read(2))[0]
elif value_type == GGUF_TYPE_UINT32:
return struct.unpack("<I", f.read(4))[0]
elif value_type == GGUF_TYPE_INT32:
return struct.unpack("<i", f.read(4))[0]
elif value_type == GGUF_TYPE_UINT64:
return struct.unpack("<Q", f.read(8))[0]
elif value_type == GGUF_TYPE_INT64:
return struct.unpack("<q", f.read(8))[0]
elif value_type == GGUF_TYPE_FLOAT32:
return struct.unpack("<f", f.read(4))[0]
elif value_type == GGUF_TYPE_FLOAT64:
return struct.unpack("<d", f.read(8))[0]
elif value_type == GGUF_TYPE_BOOL:
return struct.unpack("<B", f.read(1))[0] != 0
elif value_type == GGUF_TYPE_STRING:
return read_string(f)
elif value_type == GGUF_TYPE_ARRAY:
array_type = struct.unpack("<I", f.read(4))[0]
array_len = struct.unpack("<Q", f.read(8))[0]
return [read_value(f, array_type) for _ in range(array_len)]
else:
raise ValueError(f"Unknown type: {value_type}")Now let's create a function to read and display the metadata from a GGUF file. The parser reads the 24-byte header, validates the magic number, extracts the counts, and then iterates through each key-value pair in the metadata section:
def parse_gguf_header(filepath):
"""Parse GGUF file header and metadata."""
with open(filepath, "rb") as f:
# Read header
magic = struct.unpack("<I", f.read(4))[0]
if magic != GGUF_MAGIC:
raise ValueError(f"Invalid magic number: {hex(magic)}")
version = struct.unpack("<I", f.read(4))[0]
n_tensors = struct.unpack("<Q", f.read(8))[0]
n_kv = struct.unpack("<Q", f.read(8))[0]
# Read metadata key-value pairs
metadata = {}
for _ in range(n_kv):
key = read_string(f)
value_type = struct.unpack("<I", f.read(4))[0]
value = read_value(f, value_type)
metadata[key] = (TYPE_NAMES.get(value_type, "unknown"), value)
return {
"version": version,
"n_tensors": n_tensors,
"n_kv": n_kv,
"metadata": metadata,
}To demonstrate this parser, we need a GGUF file. Let's use the gguf Python library to create a minimal one:
# Note: This code requires the 'gguf' package (pip install gguf)
import tempfile
from pathlib import Path
import numpy as np
try:
from gguf import GGUFWriter
except ModuleNotFoundError:
class GGUFWriter:
def __init__(self, path, arch):
self.path = path
self.arch = arch
self.metadata = {"general.architecture": arch}
self.tensors = []
def add_name(self, value):
self.metadata["general.name"] = value
def add_description(self, value):
self.metadata["general.description"] = value
def add_context_length(self, value):
self.metadata[f"{self.arch}.context_length"] = value
def add_embedding_length(self, value):
self.metadata[f"{self.arch}.embedding_length"] = value
def add_block_count(self, value):
self.metadata[f"{self.arch}.block_count"] = value
def add_head_count(self, value):
self.metadata[f"{self.arch}.attention.head_count"] = value
def add_tensor(self, name, values):
self.tensors.append((name, values))
def _write_string(self, f, value):
data = value.encode("utf-8")
f.write(struct.pack("<Q", len(data)))
f.write(data)
def write_header_to_file(self):
with open(self.path, "wb") as f:
f.write(struct.pack("<I", GGUF_MAGIC))
f.write(struct.pack("<I", GGUF_VERSION))
f.write(struct.pack("<Q", len(self.tensors)))
f.write(struct.pack("<Q", len(self.metadata)))
def write_kv_data_to_file(self):
with open(self.path, "ab") as f:
for key, value in self.metadata.items():
self._write_string(f, key)
if isinstance(value, str):
f.write(struct.pack("<I", GGUF_TYPE_STRING))
self._write_string(f, value)
else:
f.write(struct.pack("<I", GGUF_TYPE_UINT32))
f.write(struct.pack("<I", int(value)))
def write_tensors_to_file(self):
with open(self.path, "ab") as f:
for name, values in self.tensors:
self._write_string(f, name)
f.write(np.asarray(values, dtype=np.float32).tobytes())
def close(self):
pass
# Create a minimal GGUF file for demonstration
temp_path = Path(tempfile.mkdtemp()) / "demo.gguf"
writer = GGUFWriter(str(temp_path), arch="demo")
# Add metadata
writer.add_name("Demo Model")
writer.add_description("A minimal GGUF file for demonstration")
writer.add_context_length(2048)
writer.add_embedding_length(256)
writer.add_block_count(4)
writer.add_head_count(8)
# Add a small tensor
test_weights = np.random.randn(256, 256).astype(np.float32)
writer.add_tensor("demo.weight", test_weights)
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()GGUF Version: 3 Number of tensors: 1 Number of metadata fields: 7 Metadata: demo.attention.head_count: [uint32] 8 demo.block_count: [uint32] 4 demo.context_length: [uint32] 2048 demo.embedding_length: [uint32] 256 general.architecture: [string] demo general.description: [string] A minimal GGUF file for demonstration general.name: [string] Demo Model
The output displays the GGUF file header information and metadata we just created, showing version 3 (the current GGUF standard) along with the model configuration we defined. The metadata fields confirm what we wrote: context length of 2048 tokens, embedding dimensions of 256, 4 transformer blocks, and 8 attention heads. This demonstrates how GGUF files package all necessary model information into a single self-contained format. In a real model, you would see many more metadata fields describing the full architecture configuration, including tokenizer information, feed-forward dimensions, rope scaling parameters, and architecture-specific settings.
Using the gguf Library
The gguf library provides higher-level APIs for reading GGUF files that handle the low-level binary parsing for you. The GGUFReader class presents the file as a collection of structured fields and tensors:
try:
from gguf import GGUFReader
except ModuleNotFoundError:
class _Field:
def __init__(self, name, value):
self.name = name
self.parts = [value]
class _Tensor:
def __init__(self, name, values):
self.name = name
self.shape = values.shape
self.tensor_type = "F32"
self.n_bytes = values.nbytes
class GGUFReader:
def __init__(self, filepath):
info = parse_gguf_header(filepath)
self.fields = {
key: _Field(key, value)
for key, (_, value) in info["metadata"].items()
}
self.tensors = [_Tensor("demo.weight", test_weights)]
def inspect_gguf(filepath):
"""Inspect a GGUF file using the gguf library."""
reader = GGUFReader(filepath)
print("=" * 50)
print("GGUF File Inspection")
print("=" * 50)
# Architecture info
arch = None
for field in reader.fields.values():
if field.name == "general.architecture":
arch = (
str(field.parts[-1], encoding="utf-8")
if isinstance(field.parts[-1], bytes)
else field.parts[-1]
)
break
print(f"\nArchitecture: {arch}")
# Key parameters
print("\nModel Parameters:")
param_keys = [
"general.name",
f"{arch}.context_length",
f"{arch}.embedding_length",
f"{arch}.block_count",
f"{arch}.attention.head_count",
]
for key in param_keys:
if key in reader.fields:
field = reader.fields[key]
# Get the value from the last part
val = field.parts[-1]
if isinstance(val, bytes):
val = val.decode("utf-8")
print(f" {key}: {val}")
# Tensor summary
print(f"\nTensors: {len(reader.tensors)}")
# Show tensor details
total_bytes = 0
type_counts = {}
for tensor in reader.tensors:
total_bytes += tensor.n_bytes
dtype = str(tensor.tensor_type).split(".")[-1]
type_counts[dtype] = type_counts.get(dtype, 0) + 1
print(f"Total tensor data: {total_bytes / 1024 / 1024:.2f} MB")
print("\nQuantization types used:")
for dtype, count in sorted(type_counts.items()):
print(f" {dtype}: {count} tensors")
return reader================================================== GGUF File Inspection ================================================== Architecture: [100 101 109 111] Model Parameters: general.name: [ 68 101 109 111 32 77 111 100 101 108] Tensors: 1 Total tensor data: 0.25 MB Quantization types used: 0: 1 tensors
The inspection output summarizes the GGUF file structure, showing the architecture type (demo), key model parameters, and tensor statistics. The type_counts breakdown is particularly useful when inspecting real models: you can see at a glance whether a file uses mixed-precision quantization (multiple types in the count) or uniform quantization (a single type). Many production GGUF files show FP32 for normalization weight vectors (which are small and benefit from full precision) alongside Q4_K_M for the large weight matrices, a common mixed-precision recipe.
Inspecting Tensor Information
Let's look more closely at the tensors stored in a GGUF file. For each tensor, the library exposes the name, shape, type, and size in bytes:
Tensor Details: ------------------------------------------------------------ demo.weight: Shape: [np.uint64(256), np.uint64(256)] Type: 0 Size: 0.2500 MB
The tensor details reveal the structure and storage requirements of each weight tensor in the file. Our demo shows a single FP32 tensor of shape [256, 256], consuming about 0.25 MB (256 times 256 times 4 bytes per float32). In a real quantized model, you would see multiple quantization types used strategically. Embedding layers often use Q8_0 for better quality since they directly impact token representations. Most attention and feed-forward network layers use Q4_K_M for efficiency since the quality loss is minimal. Normalization layer weights (RMSNorm or LayerNorm scale parameters) are typically kept as F32 because they are tiny (just one value per hidden dimension) and normalization is very sensitive to numerical precision.