if (pattern.empty()) {

return false;

}

if (pattern.back() == '*') {

pattern.remove_suffix(1);

return name.substr(0, pattern.size()) == pattern;

}

return name == pattern;

std::string_view name,

const std::vector<GgufTensorTypeOverride> & overrides) {

for (const auto & rule : overrides) {

if (tensor_type_override_matches(name, rule.pattern)) {

return rule.storage_type;

}

}

return std::nullopt;

if (dims.empty() || dims.size() > core::kMaxTensorRank) {

throw std::runtime_error("tensor rank must be between 1 and 4");

}

switch (dims.size()) {

case 1:

return core::TensorShape::from_dims({dims[0]});

case 2:

return core::TensorShape::from_dims({dims[0], dims[1]});

case 3:

return core::TensorShape::from_dims({dims[0], dims[1], dims[2]});

case 4:

return core::TensorShape::from_dims({dims[0], dims[1], dims[2], dims[3]});

default:

throw std::runtime_error("unsupported tensor rank");

}

std::string_view name,

const std::vector<int64_t> & actual_shape,

const std::optional<std::vector<int64_t>> & expected_shape) {

if (expected_shape.has_value() && actual_shape != *expected_shape) {

throw std::runtime_error("tensor shape mismatch for " + std::string(name));

}

std::string out(value);

for (char & ch : out) {

ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));

}

return out;

const std::string normalized = lower_ascii(dtype);

return (normalized == "f32" && type == GGML_TYPE_F32) ||

(normalized == "f16" && type == GGML_TYPE_F16) ||

(normalized == "bf16" && type == GGML_TYPE_BF16) ||

(normalized == "i8" && type == GGML_TYPE_I8) ||

(normalized == "q4_0" && type == GGML_TYPE_Q4_0) ||

(normalized == "q4_1" && type == GGML_TYPE_Q4_1) ||

(normalized == "q5_0" && type == GGML_TYPE_Q5_0) ||

(normalized == "q5_1" && type == GGML_TYPE_Q5_1) ||

(normalized == "q2_k" && type == GGML_TYPE_Q2_K) ||

(normalized == "q3_k" && type == GGML_TYPE_Q3_K) ||

(normalized == "q4_k" && type == GGML_TYPE_Q4_K) ||

(normalized == "q5_k" && type == GGML_TYPE_Q5_K) ||

(normalized == "q6_k" && type == GGML_TYPE_Q6_K) ||

(normalized == "q8_0" && type == GGML_TYPE_Q8_0);

const std::string normalized = lower_ascii(dtype);

if (normalized == "f32" || normalized == "float32") return GGML_TYPE_F32;

if (normalized == "f16" || normalized == "float16" || normalized == "fp16") return GGML_TYPE_F16;

if (normalized == "bf16" || normalized == "bfloat16") return GGML_TYPE_BF16;

if (normalized == "i8" || normalized == "int8") return GGML_TYPE_I8;

if (normalized == "bool" || normalized == "boolean") return GGML_TYPE_I8;

if (normalized == "i16" || normalized == "int16") return GGML_TYPE_I16;

if (normalized == "i32" || normalized == "int32") return GGML_TYPE_I32;

if (normalized == "i64" || normalized == "int64") return GGML_TYPE_I64;

if (normalized == "q4_0") return GGML_TYPE_Q4_0;

if (normalized == "q4_1") return GGML_TYPE_Q4_1;

if (normalized == "q5_0") return GGML_TYPE_Q5_0;

if (normalized == "q5_1") return GGML_TYPE_Q5_1;

if (normalized == "q8_0") return GGML_TYPE_Q8_0;

if (normalized == "q2_k") return GGML_TYPE_Q2_K;

if (normalized == "q3_k") return GGML_TYPE_Q3_K;

if (normalized == "q4_k") return GGML_TYPE_Q4_K;

if (normalized == "q5_k") return GGML_TYPE_Q5_K;

if (normalized == "q6_k") return GGML_TYPE_Q6_K;

throw std::runtime_error("unsupported tensor dtype for GGUF: " + std::string(dtype));

const char * name = ggml_type_name(type);

if (name == nullptr) {

throw std::runtime_error("GGUF tensor has an unknown ggml type");

}

return lower_ascii(name);

const size_t expected = static_cast<size_t>(shape.prefix_elements()) *

ggml_row_size(type, shape.last_dim());

if (bytes != expected) {

throw std::runtime_error("tensor byte size mismatch for " + std::string(name));

}

std::vector<std::byte> bytes(values.size() * sizeof(float));

std::memcpy(bytes.data(), values.data(), bytes.size());

return bytes;

std::vector<ggml_fp16_t> f16_values(values.size());

ggml_fp32_to_fp16_row(values.data(), f16_values.data(), static_cast<int64_t>(values.size()));

std::vector<std::byte> bytes(values.size() * sizeof(ggml_fp16_t));

std::memcpy(bytes.data(), f16_values.data(), bytes.size());

return bytes;

std::vector<ggml_bf16_t> bf16_values(values.size());

ggml_fp32_to_bf16_row(values.data(), bf16_values.data(), static_cast<int64_t>(values.size()));

std::vector<std::byte> bytes(values.size() * sizeof(ggml_bf16_t));

std::memcpy(bytes.data(), bf16_values.data(), bytes.size());

return bytes;

std::string_view name,

const std::vector<float> & values,

const core::TensorShape & shape,

ggml_type type) {

if (!ggml_is_quantized(type)) {

throw std::runtime_error("tensor quantization target is not a quantized ggml type");

}

if (ggml_quantize_requires_imatrix(type)) {

throw std::runtime_error("tensor quantization target requires an importance matrix: " + std::string(name));

}

if (shape.rank < 2) {

throw std::runtime_error("quantized tensor must have rank >= 2: " + std::string(name));

}

const int64_t elements_per_row = shape.last_dim();

if (elements_per_row % ggml_blck_size(type) != 0) {

throw std::runtime_error("quantized tensor row size is not divisible by block size: " + std::string(name));

}

const int64_t rows = shape.prefix_elements();

if (rows <= 0 || elements_per_row <= 0 ||

static_cast<size_t>(rows * elements_per_row) != values.size()) {

throw std::runtime_error("quantized tensor shape does not match F32 value count: " + std::string(name));

}

std::vector<std::byte> bytes(static_cast<size_t>(rows) * ggml_row_size(type, elements_per_row));

const size_t written = ggml_quantize_chunk(

type,

values.data(),

bytes.data(),

0,

rows,

elements_per_row,

nullptr);

if (written != bytes.size()) {

throw std::runtime_error("quantized tensor byte size mismatch: " + std::string(name));

}

return bytes;

if (tensor == nullptr) {

throw std::runtime_error("cannot upload to a null backend tensor: " + std::string(name));

}

if (bytes != ggml_nbytes(tensor)) {

throw std::runtime_error("backend tensor byte size mismatch for " + std::string(name));

}

ggml_backend_tensor_set(tensor, data, 0, bytes);

if (tensor.type == GGML_TYPE_F32) {

if (tensor.bytes.size() != static_cast<size_t>(tensor.shape.num_elements()) * sizeof(float)) {

throw std::runtime_error("invalid F32 tensor byte size: " + std::string(name));

}

std::vector<float> values(static_cast<size_t>(tensor.shape.num_elements()));

std::memcpy(values.data(), tensor.bytes.data(), tensor.bytes.size());

return values;

}

if (tensor.type == GGML_TYPE_F16) {

if (tensor.bytes.size() != static_cast<size_t>(tensor.shape.num_elements()) * sizeof(ggml_fp16_t)) {

throw std::runtime_error("invalid F16 tensor byte size: " + std::string(name));

}

std::vector<float> values(static_cast<size_t>(tensor.shape.num_elements()));

ggml_fp16_to_fp32_row(

reinterpret_cast<const ggml_fp16_t *>(tensor.bytes.data()),

values.data(),

tensor.shape.num_elements());

return values;

}

if (tensor.type == GGML_TYPE_BF16) {

if (tensor.bytes.size() != static_cast<size_t>(tensor.shape.num_elements()) * sizeof(ggml_bf16_t)) {

throw std::runtime_error("invalid BF16 tensor byte size: " + std::string(name));

}

std::vector<float> values(static_cast<size_t>(tensor.shape.num_elements()));

ggml_bf16_to_fp32_row(

reinterpret_cast<const ggml_bf16_t *>(tensor.bytes.data()),

values.data(),

tensor.shape.num_elements());

return values;

}

const ggml_type_traits * traits = ggml_get_type_traits(tensor.type);

if (traits == nullptr || traits->to_float == nullptr) {

throw std::runtime_error("tensor type is not readable as F32 data: " + std::string(name));

}

if (tensor.shape.rank < 2) {

throw std::runtime_error("quantized tensor must have rank >= 2: " + std::string(name));

}

const int64_t cols = tensor.shape.last_dim();

const int64_t rows = tensor.shape.prefix_elements();

const size_t row_bytes = ggml_row_size(tensor.type, cols);

if (tensor.bytes.size() != static_cast<size_t>(rows) * row_bytes) {

throw std::runtime_error("quantized tensor byte size mismatch: " + std::string(name));

}

std::vector<float> values(static_cast<size_t>(rows * cols));

const std::byte * src = tensor.bytes.data();

for (int64_t row = 0; row < rows; ++row) {

traits->to_float(

src + static_cast<std::ptrdiff_t>(row * static_cast<int64_t>(row_bytes)),

values.data() + static_cast<std::ptrdiff_t>(row * cols),

cols);

}

return values;

ggml_tensor * tensor,

std::string_view name,

const std::vector<float> & values,

const core::TensorShape & shape,

ggml_type type) {

if (type == GGML_TYPE_F32) {

set_tensor_bytes(tensor, values.data(), values.size() * sizeof(float), name);

return;

}

if (type == GGML_TYPE_F16) {

const auto bytes = f16_bytes(values);

set_tensor_bytes(tensor, bytes.data(), bytes.size(), name);

return;

}

if (type == GGML_TYPE_BF16) {

const auto bytes = bf16_bytes(values);

set_tensor_bytes(tensor, bytes.data(), bytes.size(), name);

return;

}

const auto bytes = quantize_f32_rows(name, values, shape, type);

set_tensor_bytes(tensor, bytes.data(), bytes.size(), name);

ggml_tensor * tensor,

std::string_view name,

const std::vector<float> & values,

const core::TensorShape & shape,

ggml_type type) {

if (static_cast<int64_t>(values.size()) < kParallelF32ConvertElements ||

(type != GGML_TYPE_F16 && type != GGML_TYPE_BF16)) {

set_backend_tensor_from_f32(tensor, name, values, shape, type);

return;

}

if (type == GGML_TYPE_F16) {

std::vector<ggml_fp16_t> converted(values.size());

const int64_t count = static_cast<int64_t>(values.size());

const int64_t chunks = (count + kF32ConvertChunkElements - 1) / kF32ConvertChunkElements;

#pragma omp parallel for schedule(static)

for (int64_t chunk = 0; chunk < chunks; ++chunk) {

const int64_t offset = chunk * kF32ConvertChunkElements;

const int64_t length = std::min(kF32ConvertChunkElements, count - offset);

ggml_fp32_to_fp16_row(values.data() + offset, converted.data() + offset, length);

}

set_tensor_bytes(tensor, converted.data(), converted.size() * sizeof(ggml_fp16_t), name);

return;

}

std::vector<ggml_bf16_t> converted(values.size());

const int64_t count = static_cast<int64_t>(values.size());

const int64_t chunks = (count + kF32ConvertChunkElements - 1) / kF32ConvertChunkElements;

#pragma omp parallel for schedule(static)

for (int64_t chunk = 0; chunk < chunks; ++chunk) {

const int64_t offset = chunk * kF32ConvertChunkElements;

const int64_t length = std::min(kF32ConvertChunkElements, count - offset);

ggml_fp32_to_bf16_row(values.data() + offset, converted.data() + offset, length);

}

set_tensor_bytes(tensor, converted.data(), converted.size() * sizeof(ggml_bf16_t), name);

std::string_view name,

const std::vector<float> & values,

const core::TensorShape & shape,

ggml_type type) {

if (type == GGML_TYPE_F32) {

return f32_bytes(values);

}

if (type == GGML_TYPE_F16) {

return f16_bytes(values);

}

if (type == GGML_TYPE_BF16) {

return bf16_bytes(values);

}

return quantize_f32_rows(name, values, shape, type);

int64_t count = 1;

for (const int64_t dim : shape) {

if (dim <= 0) {

throw std::runtime_error("tensor shape contains a non-positive dimension: " + std::string(name));

}

if (count > std::numeric_limits<int64_t>::max() / dim) {

throw std::runtime_error("tensor element count overflow: " + std::string(name));

}

count *= dim;

}

return count;

if (!dtype_equals(data.metadata.dtype, "U8")) {

throw std::runtime_error("BNB NF4 tensor dtype mismatch for " + data.metadata.name);

}

std::vector<uint8_t> values(data.bytes.size());

std::memcpy(values.data(), data.bytes.data(), data.bytes.size());

return values;

if (!dtype_equals(data.metadata.dtype, "F32")) {

throw std::runtime_error("BNB NF4 tensor dtype mismatch for " + data.metadata.name);

}

if (data.bytes.size() % sizeof(float) != 0) {

throw std::runtime_error("BNB NF4 F32 tensor byte size mismatch: " + data.metadata.name);

}

std::vector<float> values(data.bytes.size() / sizeof(float));

std::memcpy(values.data(), data.bytes.data(), data.bytes.size());

return values;

std::vector<int64_t> shape;

int64_t blocksize = 0;

int64_t nested_blocksize = 0;

float nested_offset = 0.0F;

const auto bytes = raw_u8_values(data);

const auto root = engine::io::json::parse(std::string(reinterpret_cast<const char *>(bytes.data()), bytes.size()));

BnbNf4QuantState state;

const auto quant_type = engine::io::json::require_string(root, "quant_type");

const auto dtype = engine::io::json::require_string(root, "dtype");

state.shape = engine::io::json::require_i64_array(root, "shape");

state.blocksize = engine::io::json::require_i64(root, "blocksize");

state.nested_blocksize = engine::io::json::require_i64(root, "nested_blocksize");

state.nested_offset = engine::io::json::require_f32(root, "nested_offset");

if (quant_type != "nf4") {

throw std::runtime_error("BNB quant_state is not NF4: " + data.metadata.name);

}

if (state.blocksize <= 0 || state.nested_blocksize <= 0) {

throw std::runtime_error("BNB NF4 quant_state contains invalid block sizes: " + data.metadata.name);

}

if (dtype != "bfloat16" && dtype != "float16" && dtype != "float32") {

throw std::runtime_error("BNB NF4 quant_state contains unsupported dequant dtype: " + dtype);

}

return state;

static constexpr std::string_view suffixes[] = {

".absmax",

".nested_absmax",

".nested_quant_map",

".quant_map",

".quant_state.bitsandbytes__nf4",

};

const std::string value(name);

for (const auto suffix : suffixes) {

if (value.size() <= suffix.size() ||

value.compare(value.size() - suffix.size(), suffix.size(), suffix) != 0) {

continue;

}

const std::string base = value.substr(0, value.size() - suffix.size());

if (source.has_tensor(base) && source.has_tensor(bnb_nf4_quant_state_name(base))) {

return base;

}

}

return std::nullopt;

const TensorSource & source,

const TensorMetadata & metadata,

TensorStorageType storage_type) {

const ggml_type output_type = ggml_type_for_tensor_storage(storage_type);

const auto packed = source.require_tensor_data(metadata.name);

const auto state = parse_bnb_nf4_quant_state(source.require_tensor_data(bnb_nf4_quant_state_name(metadata.name)));

const int64_t elements = checked_element_count(metadata.name, state.shape);

if (packed.metadata.shape != std::vector<int64_t>{(elements + 1) / 2, 1}) {

throw std::runtime_error("BNB NF4 packed tensor shape mismatch: " + metadata.name);

}

const auto absmax = raw_u8_values(source.require_tensor_data(std::string(metadata.name) + ".absmax"));

const auto nested_absmax = raw_f32_values(source.require_tensor_data(std::string(metadata.name) + ".nested_absmax"));

const auto nested_quant_map =

raw_f32_values(source.require_tensor_data(std::string(metadata.name) + ".nested_quant_map"));

const auto quant_map = raw_f32_values(source.require_tensor_data(std::string(metadata.name) + ".quant_map"));

if (nested_quant_map.size() != 256 || quant_map.size() != 16) {

throw std::runtime_error("BNB NF4 quant maps have unexpected size: " + metadata.name);

}

const int64_t expected_nested =

(static_cast<int64_t>(absmax.size()) + state.nested_blocksize - 1) / state.nested_blocksize;

if (static_cast<int64_t>(nested_absmax.size()) != expected_nested) {

throw std::runtime_error("BNB NF4 nested absmax length mismatch: " + metadata.name);

}

std::vector<float> scales(absmax.size());

for (size_t i = 0; i < absmax.size(); ++i) {

scales[i] = nested_quant_map[absmax[i]] *

nested_absmax[static_cast<size_t>(static_cast<int64_t>(i) / state.nested_blocksize)] +

state.nested_offset;

}

const auto packed_values = raw_u8_values(packed);

if (static_cast<int64_t>(packed_values.size()) != (elements + 1) / 2) {

throw std::runtime_error("BNB NF4 packed byte count mismatch: " + metadata.name);

}

const int64_t block_count = (elements + state.blocksize - 1) / state.blocksize;

if (static_cast<int64_t>(scales.size()) != block_count) {

throw std::runtime_error("BNB NF4 absmax block count mismatch: " + metadata.name);

}

std::vector<float> values(static_cast<size_t>(elements));

int64_t out = 0;

for (const uint8_t byte : packed_values) {

values[static_cast<size_t>(out)] =

quant_map[(byte >> 4U) & 0x0FU] * scales[static_cast<size_t>(out / state.blocksize)];

++out;

if (out < elements) {

values[static_cast<size_t>(out)] =

quant_map[byte & 0x0FU] * scales[static_cast<size_t>(out / state.blocksize)];

++out;

}

}

RawTensorData out_data;

out_data.metadata = {metadata.name, dtype_for_ggml_type(output_type), state.shape};

out_data.bytes = encode_f32_tensor_data(metadata.name, values, shape_from_dims(state.shape), output_type);

return out_data;

explicit SafeTensorSource(std::filesystem::path path)

: index_(engine::io::load_safetensors_index(path)),

bytes_(engine::io::read_binary_blob(path)) {}

const std::filesystem::path & source_path() const noexcept override {

return index_.source_path;

}

bool has_tensor(std::string_view name) const noexcept override {

return index_.tensors.find(std::string(name)) != index_.tensors.end();

}

TensorMetadata require_metadata(std::string_view name) const override {

const auto * info = find_info(name);

if (info == nullptr) {

throw std::runtime_error("missing tensor: " + std::string(name));

}

return TensorMetadata{info->name, info->dtype, info->shape};

}

std::vector<TensorMetadata> tensors() const override {

std::vector<TensorMetadata> out;

out.reserve(index_.tensors.size());

for (const auto & [name, info] : index_.tensors) {

out.push_back({name, info.dtype, info.shape});

}

std::sort(out.begin(), out.end(), [](const TensorMetadata & lhs, const TensorMetadata & rhs) {

return lhs.name < rhs.name;

});

return out;

}

void release_storage() const override {

bytes_ = engine::io::BinaryBlob();

}

RawTensorData require_tensor_data(std::string_view name) const override {

const auto * info = find_info(name);

if (info == nullptr) {

throw std::runtime_error("missing tensor: " + std::string(name));

}

const auto [data, byte_size] = require_data_range(*info);

RawTensorData tensor;

tensor.metadata = TensorMetadata{info->name, info->dtype, info->shape};

tensor.bytes.resize(byte_size);

std::memcpy(tensor.bytes.data(), data, byte_size);

discard_data_range(*info);

return tensor;

}

void set_backend_tensor(

ggml_tensor * tensor,

std::string_view name,

TensorStorageType storage_type,

const std::vector<int64_t> & expected_shape) const override {

const auto * info = find_info(name);

if (info == nullptr) {

throw std::runtime_error("missing tensor: " + std::string(name));

}

validate_expected_shape(name, info->shape, expected_shape);

const auto shape = shape_from_dims(expected_shape);

const ggml_type type = ggml_type_for_tensor_storage(resolve_tensor_storage_type(*this, name, storage_type));

const auto [data, byte_size] = require_data_range(*info);

if (raw_dtype_matches_ggml_type(info->dtype, type)) {

validate_raw_tensor_byte_size(name, shape, type, byte_size);

set_tensor_bytes(tensor, data, byte_size, name);

discard_data_range(*info);

return;

}

const ggml_type raw_type = ggml_type_for_tensor_storage(tensor_storage_type_for_dtype(info->dtype));

std::vector<std::byte> raw_bytes(byte_size);

std::memcpy(raw_bytes.data(), data, byte_size);

const auto values = decode_tensor_data_f32(name, TensorData{shape, raw_type, std::move(raw_bytes)});

set_backend_tensor_from_f32(tensor, name, values, shape, type);

discard_data_range(*info);

}

void set_backend_f32_tensor(

ggml_tensor * tensor,

std::string_view name,

const std::vector<int64_t> & expected_shape) const override {

set_backend_tensor(tensor, name, TensorStorageType::F32, expected_shape);

}

std::vector<float> require_f32(

std::string_view name,

const std::optional<std::vector<int64_t>> & expected_shape) const override {

const auto tensor = require_tensor_data(name);

validate_expected_shape(name, tensor.metadata.shape, expected_shape);

const ggml_type type = ggml_type_for_tensor_dtype(tensor.metadata.dtype);

const auto physical_shape = tensor.metadata.shape.empty()

? shape_from_dims({1})

: shape_from_dims(tensor.metadata.shape);

return decode_tensor_data_f32(name, TensorData{physical_shape, type, tensor.bytes});

}

std::optional<std::vector<float>> optional_f32(

std::string_view name,

const std::optional<std::vector<int64_t>> & expected_shape) const override {

if (!has_tensor(name)) {

return std::nullopt;

}

return require_f32(name, expected_shape);

}

int64_t require_i64_scalar(std::string_view name) const override {

const auto * info = find_info(name);

if (info == nullptr) {

throw std::runtime_error("missing tensor: " + std::string(name));

}

if (info->dtype != "I64" || info->data_end - info->data_begin != sizeof(int64_t)) {

throw std::runtime_error("tensor is not an I64 scalar: " + std::string(name));

}

const auto [data, byte_size] = require_data_range(*info);

(void) byte_size;

int64_t value = 0;

std::memcpy(&value, data, sizeof(value));

return value;

}

const engine::io::SafeTensorInfo * find_info(std::string_view name) const noexcept {

const auto it = index_.tensors.find(std::string(name));

if (it == index_.tensors.end()) {

return nullptr;

}

return &it->second;

}

std::pair<const std::byte *, size_t> require_data_range(const engine::io::SafeTensorInfo & info) const {

if (bytes_.empty()) {

bytes_ = engine::io::read_binary_blob(index_.source_path);

}

const size_t byte_size = info.data_end - info.data_begin;

// Written as subtractions against the total so nothing can overflow.

// `header_bytes + data_begin + byte_size > total` wraps if any term is

// large, and a wrapped comparison passes -- handing back a span over

// memory past the end of the blob. Parse-time validation now rejects the

// inputs that made that reachable; this is the last gate before a raw

// pointer escapes, so it checks anyway.

const size_t total = bytes_.size();

if (index_.header_bytes > total || byte_size > total - index_.header_bytes ||

info.data_begin > total - index_.header_bytes - byte_size) {

throw std::runtime_error("tensor data range is out of bounds: " + info.name);

}

const size_t data_offset = index_.header_bytes + info.data_begin;

return {bytes_.data() + static_cast<std::ptrdiff_t>(data_offset), byte_size};

}

void discard_data_range(const engine::io::SafeTensorInfo & info) const noexcept {

bytes_.discard_range(index_.header_bytes + info.data_begin, info.data_end - info.data_begin);

}

engine::io::SafeTensorIndex index_;

mutable engine::io::BinaryBlob bytes_;

std::string logical_name;

std::string physical_name;

std::string dtype;

std::vector<int64_t> shape;

ggml_type type = GGML_TYPE_F32;

size_t data_offset = 0;

size_t byte_size = 0;

explicit GgufTensorSource(std::filesystem::path path)

: source_path_(std::filesystem::weakly_canonical(path)) {

ggml_context * tensor_context = nullptr;

gguf_context * gguf = gguf_init_from_file(

source_path_.string().c_str(),

gguf_init_params{true, &tensor_context});

if (gguf == nullptr || tensor_context == nullptr) {

if (gguf != nullptr) gguf_free(gguf);

if (tensor_context != nullptr) ggml_free(tensor_context);

throw std::runtime_error("failed to read GGUF tensor source: " + source_path_.string());

}

try {

data_begin_ = gguf_get_data_offset(gguf);

const int64_t logical_names_key = gguf_find_key(gguf, "audiocpp.tensor_names");

if (logical_names_key >= 0 &&

(gguf_get_kv_type(gguf, logical_names_key) != GGUF_TYPE_ARRAY ||

gguf_get_arr_type(gguf, logical_names_key) != GGUF_TYPE_STRING ||

gguf_get_arr_n(gguf, logical_names_key) != static_cast<size_t>(gguf_get_n_tensors(gguf)))) {

throw std::runtime_error("GGUF audiocpp.tensor_names metadata is invalid");

}

const int64_t count = gguf_get_n_tensors(gguf);

const int64_t ranks_key = gguf_find_key(gguf, "audiocpp.tensor_ranks");

const int64_t shapes_key = gguf_find_key(gguf, "audiocpp.tensor_shapes");

const bool has_exact_shapes = ranks_key >= 0 && shapes_key >= 0;

if ((ranks_key >= 0) != (shapes_key >= 0) ||

(has_exact_shapes &&

(gguf_get_kv_type(gguf, ranks_key) != GGUF_TYPE_ARRAY ||

gguf_get_arr_type(gguf, ranks_key) != GGUF_TYPE_INT32 ||

gguf_get_arr_n(gguf, ranks_key) != static_cast<size_t>(count) ||

gguf_get_kv_type(gguf, shapes_key) != GGUF_TYPE_ARRAY ||

gguf_get_arr_type(gguf, shapes_key) != GGUF_TYPE_INT64))) {

throw std::runtime_error("GGUF audiocpp exact tensor shape metadata is invalid");

}

size_t shape_cursor = 0;

infos_.reserve(static_cast<size_t>(count));

for (int64_t i = 0; i < count; ++i) {

GgufTensorInfo info;

info.physical_name = gguf_get_tensor_name(gguf, i);

info.logical_name = logical_names_key >= 0

? gguf_get_arr_str(gguf, logical_names_key, static_cast<size_t>(i))

: info.physical_name;

info.type = gguf_get_tensor_type(gguf, i);

info.dtype = dtype_for_ggml_type(info.type);

info.data_offset = gguf_get_tensor_offset(gguf, i);

info.byte_size = gguf_get_tensor_size(gguf, i);

const ggml_tensor * tensor = ggml_get_tensor(tensor_context, info.physical_name.c_str());

if (tensor == nullptr) {

throw std::runtime_error("GGUF tensor metadata is missing: " + info.physical_name);

}

if (has_exact_shapes) {

const auto * ranks = static_cast<const int32_t *>(gguf_get_arr_data(gguf, ranks_key));

const auto * shapes = static_cast<const int64_t *>(gguf_get_arr_data(gguf, shapes_key));

const int32_t tensor_rank = ranks[i];

if (tensor_rank < 0 || tensor_rank > static_cast<int32_t>(core::kMaxTensorRank) ||

shape_cursor + static_cast<size_t>(tensor_rank) > gguf_get_arr_n(gguf, shapes_key)) {

throw std::runtime_error("GGUF contains invalid exact tensor dimensions");

}

if (tensor_rank > 0) {

info.shape.assign(shapes + shape_cursor, shapes + shape_cursor + tensor_rank);

} else {

info.shape.clear();

}

shape_cursor += static_cast<size_t>(tensor_rank);

} else {

const int dimensions = ggml_n_dims(tensor);

info.shape.reserve(static_cast<size_t>(dimensions));

for (int dim = dimensions - 1; dim >= 0; --dim) {

info.shape.push_back(tensor->ne[dim]);

}

}

if (!info_by_name_.emplace(info.logical_name, infos_.size()).second) {

throw std::runtime_error("GGUF contains duplicate logical tensor name: " + info.logical_name);

}

infos_.push_back(std::move(info));

}

if (has_exact_shapes && shape_cursor != gguf_get_arr_n(gguf, shapes_key)) {

throw std::runtime_error("GGUF exact tensor shape metadata has trailing dimensions");

}

} catch (...) {

gguf_free(gguf);

ggml_free(tensor_context);

throw;

}

gguf_free(gguf);

ggml_free(tensor_context);

bytes_ = engine::io::read_binary_blob(source_path_);

}

const std::filesystem::path & source_path() const noexcept override { return source_path_; }

bool has_tensor(std::string_view name) const noexcept override {

return info_by_name_.find(std::string(name)) != info_by_name_.end();

}

TensorMetadata require_metadata(std::string_view name) const override {

const auto & info = require_info(name);

return {info.logical_name, info.dtype, info.shape};

}

std::vector<TensorMetadata> tensors() const override {

std::vector<TensorMetadata> out;

out.reserve(infos_.size());

for (const auto & info : infos_) out.push_back({info.logical_name, info.dtype, info.shape});

std::sort(out.begin(), out.end(), [](const TensorMetadata & lhs, const TensorMetadata & rhs) {

return lhs.name < rhs.name;

});

return out;

}

void release_storage() const override { bytes_ = engine::io::BinaryBlob(); }

RawTensorData require_tensor_data(std::string_view name) const override {

const auto & info = require_info(name);

const auto [data, byte_size] = require_data_range(info);

RawTensorData out;

out.metadata = {info.logical_name, info.dtype, info.shape};

out.bytes.resize(byte_size);

std::memcpy(out.bytes.data(), data, byte_size);

bytes_.discard_range(data_begin_ + info.data_offset, byte_size);

return out;

}

void set_backend_tensor(

ggml_tensor * tensor,

std::string_view name,

TensorStorageType storage_type,

const std::vector<int64_t> & expected_shape) const override {

const auto & info = require_info(name);

validate_expected_shape(name, info.shape, expected_shape);

const auto shape = shape_from_dims(expected_shape);

const ggml_type type = ggml_type_for_tensor_storage(resolve_tensor_storage_type(*this, name, storage_type));

const auto [data, byte_size] = require_data_range(info);

if (info.type == type) {

validate_raw_tensor_byte_size(name, shape, type, byte_size);

set_tensor_bytes(tensor, data, byte_size, name);

bytes_.discard_range(data_begin_ + info.data_offset, byte_size);

return;

}

std::vector<std::byte> raw_bytes(byte_size);

std::memcpy(raw_bytes.data(), data, byte_size);

const auto values = decode_tensor_data_f32(name, TensorData{shape, info.type, std::move(raw_bytes)});

set_backend_tensor_from_f32(tensor, name, values, shape, type);

bytes_.discard_range(data_begin_ + info.data_offset, byte_size);

}

void set_backend_f32_tensor(

ggml_tensor * tensor,

std::string_view name,

const std::vector<int64_t> & expected_shape) const override {

set_backend_tensor(tensor, name, TensorStorageType::F32, expected_shape);

}

std::vector<float> require_f32(

std::string_view name,

const std::optional<std::vector<int64_t>> & expected_shape) const override {

const auto tensor = require_tensor_data(name);

validate_expected_shape(name, tensor.metadata.shape, expected_shape);

const auto physical_shape = tensor.metadata.shape.empty()

? shape_from_dims({1})

: shape_from_dims(tensor.metadata.shape);

return decode_tensor_data_f32(

name,

TensorData{physical_shape, require_info(name).type, tensor.bytes});

}

std::optional<std::vector<float>> optional_f32(

std::string_view name,

const std::optional<std::vector<int64_t>> & expected_shape) const override {

if (!has_tensor(name)) return std::nullopt;

return require_f32(name, expected_shape);

}

int64_t require_i64_scalar(std::string_view name) const override {

const auto & info = require_info(name);

if (info.type != GGML_TYPE_I64 || info.byte_size != sizeof(int64_t)) {

throw std::runtime_error("tensor is not an I64 scalar: " + std::string(name));

}

const auto [data, byte_size] = require_data_range(info);

(void) byte_size;

int64_t value = 0;

std::memcpy(&value, data, sizeof(value));

return value;

}

const GgufTensorInfo & require_info(std::string_view name) const {

const auto it = info_by_name_.find(std::string(name));

if (it == info_by_name_.end()) throw std::runtime_error("missing tensor: " + std::string(name));

return infos_[it->second];

}

std::pair<const std::byte *, size_t> require_data_range(const GgufTensorInfo & info) const {

if (bytes_.empty()) bytes_ = engine::io::read_binary_blob(source_path_);

const size_t offset = data_begin_ + info.data_offset;

if (offset > bytes_.size() || info.byte_size > bytes_.size() - offset) {

throw std::runtime_error("GGUF tensor data range is out of bounds: " + info.logical_name);

}

return {bytes_.data() + static_cast<std::ptrdiff_t>(offset), info.byte_size};

}

std::filesystem::path source_path_;

size_t data_begin_ = 0;

std::vector<GgufTensorInfo> infos_;

std::unordered_map<std::string, size_t> info_by_name_;

mutable engine::io::BinaryBlob bytes_;

const std::filesystem::path & index_path) {

const auto root = engine::io::json::parse_file(index_path);

const auto & object = root.require("weight_map").as_object();

std::unordered_map<std::string, std::string> weight_map;

weight_map.reserve(object.size());

for (const auto & [name, value] : object) {

weight_map.emplace(name, value.as_string());

}

if (weight_map.empty()) {

throw std::runtime_error("indexed tensor source has an empty weight_map: " + index_path.string());

}

return weight_map;

const std::filesystem::path & model_root,

const std::unordered_map<std::string, std::string> & weight_map) {

std::set<std::string> shard_names;

for (const auto & [_, file_name] : weight_map) {

shard_names.insert(file_name);

}

std::vector<std::filesystem::path> paths;

paths.reserve(shard_names.size());

for (const auto & name : shard_names) {

const auto path = model_root / name;

if (!engine::io::is_existing_file(path)) {

throw std::runtime_error("missing indexed tensor shard: " + path.string());

}

paths.push_back(std::filesystem::weakly_canonical(path));

}

return paths;

IndexedTensorSource(

std::filesystem::path index_path,

std::unordered_map<std::string, std::string> weight_map,

std::unordered_map<std::string, std::shared_ptr<const TensorSource>> shard_sources)

: index_path_(std::move(index_path)),

weight_map_(std::move(weight_map)),

shard_sources_(std::move(shard_sources)) {}

const std::filesystem::path & source_path() const noexcept override {

return index_path_;

}

bool has_tensor(std::string_view name) const noexcept override {

const auto route = weight_map_.find(std::string(name));

if (route == weight_map_.end()) {

return false;

}

const auto source = shard_sources_.find(route->second);

return source != shard_sources_.end() && source->second->has_tensor(name);

}

TensorMetadata require_metadata(std::string_view name) const override {

return source_for(name)->require_metadata(name);

}

std::vector<TensorMetadata> tensors() const override {

std::vector<TensorMetadata> out;

out.reserve(weight_map_.size());

for (const auto & [name, _] : weight_map_) {

out.push_back(require_metadata(name));

}

std::sort(out.begin(), out.end(), [](const TensorMetadata & lhs, const TensorMetadata & rhs) {

return lhs.name < rhs.name;

});

return out;

}

void release_storage() const override {

for (const auto & [_, source] : shard_sources_) {

source->release_storage();

}

}

RawTensorData require_tensor_data(std::string_view name) const override {

return source_for(name)->require_tensor_data(name);

}

std::vector<float> require_f32(

std::string_view name,

const std::optional<std::vector<int64_t>> & expected_shape) const override {

return source_for(name)->require_f32(name, expected_shape);

}

std::optional<std::vector<float>> optional_f32(

std::string_view name,

const std::optional<std::vector<int64_t>> & expected_shape) const override {

if (!has_tensor(name)) {

return std::nullopt;

}

return require_f32(name, expected_shape);

}

void set_backend_tensor(

ggml_tensor * tensor,

std::string_view name,

TensorStorageType storage_type,

const std::vector<int64_t> & expected_shape) const override {

source_for(name)->set_backend_tensor(tensor, name, storage_type, expected_shape);

}

void set_backend_f32_tensor(

ggml_tensor * tensor,

std::string_view name,

const std::vector<int64_t> & expected_shape) const override {

source_for(name)->set_backend_f32_tensor(tensor, name, expected_shape);

}

int64_t require_i64_scalar(std::string_view name) const override {

return source_for(name)->require_i64_scalar(name);

}