const uint32_t exponent_bias = 7;

if (f8 == 0xff) {

return ggml_fp32_to_fp16(-NAN);

} else if (f8 == 0x7f) {

return ggml_fp32_to_fp16(NAN);

}

uint32_t sign = f8 & 0x80;

uint32_t exponent = (f8 & 0x78) >> 3;

uint32_t mantissa = f8 & 0x07;

uint32_t result = sign << 24;

if (exponent == 0) {

if (mantissa > 0) {

exponent = 0x7f - exponent_bias;

// yes, 2 times

if ((mantissa & 0x04) == 0) {

mantissa &= 0x03;

mantissa <<= 1;

exponent -= 1;

}

if ((mantissa & 0x04) == 0) {

mantissa &= 0x03;

mantissa <<= 1;

exponent -= 1;

}

result |= (mantissa & 0x03) << 21;

result |= exponent << 23;

}

} else {

result |= mantissa << 20;

exponent += 0x7f - exponent_bias;

result |= exponent << 23;

}

return ggml_fp32_to_fp16(*reinterpret_cast<const float*>(&result));

// support inplace op

for (int64_t i = n - 1; i >= 0; i--) {

dst[i] = f8_e4m3_to_f16(src[i]);

}

// support inplace op

for (int64_t i = n - 1; i >= 0; i--) {

dst[i] = f8_e5m2_to_f16(src[i]);

}

// support inplace op

for (int64_t i = 0; i < n; i++) {

dst[i] = (float)src[i];

}

// support inplace op

for (int64_t i = 0; i < n; i++) {

dst[i] = (int32_t)src[i];

}

ggml_type src_type,

void* dst,

ggml_type dst_type,

int nrows,

int n_per_row,

std::vector<float> imatrix = {}) {

int n = nrows * n_per_row;

if (src_type == dst_type) {

size_t nbytes = n * ggml_type_size(src_type) / ggml_blck_size(src_type);

memcpy(((char*)dst), ((char*)src), nbytes);

} else if (src_type == GGML_TYPE_F32) {

if (dst_type == GGML_TYPE_F16) {

ggml_fp32_to_fp16_row((float*)src, (ggml_fp16_t*)dst, n);

} else {

imatrix.resize(n_per_row, 1.0f);

const float* im = imatrix.data();

ggml_quantize_chunk(dst_type, (float*)src, dst, 0, nrows, n_per_row, im);

}

} else if (dst_type == GGML_TYPE_F32) {

if (src_type == GGML_TYPE_F16) {

ggml_fp16_to_fp32_row((ggml_fp16_t*)src, (float*)dst, n);

} else {

auto qtype = ggml_get_type_traits(src_type);

if (qtype->to_float == nullptr) {

throw std::runtime_error(sd_format("type %s unsupported for integer quantization: no dequantization available",

ggml_type_name(src_type)));

}

qtype->to_float(src, (float*)dst, n);

}

} else {

// src_type == GGML_TYPE_F16 => dst_type is quantized

// src_type is quantized => dst_type == GGML_TYPE_F16 or dst_type is quantized

auto qtype = ggml_get_type_traits(src_type);

if (qtype->to_float == nullptr) {

throw std::runtime_error(sd_format("type %s unsupported for integer quantization: no dequantization available",

ggml_type_name(src_type)));

}

std::vector<char> buf;

buf.resize(sizeof(float) * n);

char* src_data_f32 = buf.data();

qtype->to_float(src, (float*)src_data_f32, n);

if (dst_type == GGML_TYPE_F16) {

ggml_fp32_to_fp16_row((float*)src_data_f32, (ggml_fp16_t*)dst, n);

} else {

imatrix.resize(n_per_row, 1.0f);

const float* im = imatrix.data();

ggml_quantize_chunk(dst_type, (float*)src_data_f32, dst, 0, nrows, n_per_row, im);

}

}

auto it = std::find(file_paths_.begin(), file_paths_.end(), file_path);

if (it != file_paths_.end()) {

return static_cast<size_t>(it - file_paths_.begin());

}

invalidate_file_data();

file_paths_.push_back(file_path);

return file_paths_.size() - 1;

if (tensor_storage_map.count(tensor_storage.name) != 0) {

throw std::runtime_error("duplicate tensor in model source: " + tensor_storage.name);

}

tensor_storage_map[tensor_storage.name] = tensor_storage;

FileStamp stamp;

if (!read_file_stamp(file_path, stamp)) {

return false;

}

parsed_dependencies_.push_back(stamp);

if (is_directory(file_path)) {

const std::string diffusers_index_path = path_join(file_path, "model_index.json");

const std::string diffusers_unet_path = path_join(file_path, "unet/diffusion_pytorch_model.safetensors");

const bool has_diffusers_layout = file_exists(diffusers_index_path) || file_exists(diffusers_unet_path);

const std::string safetensors_index_path = path_join(file_path, "model.safetensors.index.json");

if (!has_diffusers_layout && file_exists(safetensors_index_path)) {

LOG_INFO("load %s using root safetensors index", file_path.c_str());

return parse_file(safetensors_index_path, prefix);

}

LOG_INFO("load %s using diffusers format", file_path.c_str());

return init_from_diffusers_file(file_path, prefix);

} else if (is_gguf_file(file_path)) {

LOG_INFO("load %s using gguf format", file_path.c_str());

return init_from_gguf_file(file_path, prefix);

} else if (ends_with(file_path, ".json")) {

LOG_INFO("load %s using safetensors index format", file_path.c_str());

return init_from_safetensors_index_file(file_path, prefix);

} else if (is_safetensors_file(file_path)) {

LOG_INFO("load %s using safetensors format", file_path.c_str());

return init_from_safetensors_file(file_path, prefix);

} else if (is_torch_zip_file(file_path)) {

LOG_INFO("load %s using torch zip format", file_path.c_str());

return init_from_torch_zip_file(file_path, prefix);

} else if (init_from_torch_legacy_file(file_path, prefix)) {

LOG_INFO("load %s using torch legacy format", file_path.c_str());

return true;

} else {

if (file_exists(file_path)) {

LOG_WARN("unknown format %s", file_path.c_str());

} else {

LOG_WARN("file %s not found", file_path.c_str());

}

return false;

}

if (names_converted_) {

return;

}

names_converted_ = true;

rebuild_catalog();

if (version_ == VERSION_COUNT && version != VERSION_COUNT) {

version_ = version;

}

if (!init_from_file(file_path, prefix)) {

return false;

}

convert_tensors_name();

return true;

LOG_VERBOSE("init from '%s'", file_path.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_gguf_file(file_path, tensor_storages, &error)) {

LOG_ERROR("%s", error.c_str());

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

// LOG_VERBOSE("%s", tensor_storage.name.c_str());

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

}

return true;

LOG_VERBOSE("init from '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_safetensors_file(file_path, tensor_storages, &error, &metadata_, &parsed_tensor_names_[file_path])) {

LOG_ERROR("%s", error.c_str());

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

// LOG_VERBOSE("%s", tensor_storage.to_string().c_str());

}

return true;

LOG_VERBOSE("init from safetensors index '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());

std::vector<std::string> shard_paths;

std::string error;

if (!read_safetensors_index_file(file_path, shard_paths, &error)) {

LOG_ERROR("%s", error.c_str());

return false;

}

for (const std::string& shard_path : shard_paths) {

FileStamp stamp;

if (!read_file_stamp(shard_path, stamp)) {

return false;

}

parsed_dependencies_.push_back(stamp);

LOG_INFO("load %s using safetensors format", shard_path.c_str());

if (!init_from_safetensors_file(shard_path, prefix)) {

return false;

}

}

std::ifstream index_file(file_path);

const auto index = nlohmann::json::parse(index_file);

for (const auto& entry : index.at("weight_map").items()) {

const auto expected = (std::filesystem::u8path(file_path).parent_path() /

std::filesystem::u8path(entry.value().get<std::string>()))

.lexically_normal();

bool found = false;

for (const auto& shard : parsed_tensor_names_) {

if (std::filesystem::u8path(shard.first).lexically_normal() == expected) {

found = shard.second.count(entry.key()) != 0;

break;

}

}

if (!found) {

LOG_ERROR("safetensors index tensor '%s' is missing from its declared shard", entry.key().c_str());

return false;

}

}

return true;

LOG_VERBOSE("init from torch legacy '%s'", file_path.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_torch_legacy_file(file_path, tensor_storages, &error)) {

if ((!error.empty()) && (ends_with(file_path, ".pt") || ends_with(file_path, ".pth"))) {

LOG_WARN("%s", error.c_str());

}

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

}

return true;

LOG_VERBOSE("init from '%s'", file_path.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_torch_zip_file(file_path, tensor_storages, &error)) {

LOG_ERROR("%s", error.c_str());

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

// LOG_VERBOSE("%s", tensor_storage.to_string().c_str());

}

return true;

std::string unet_path = path_join(file_path, "unet/diffusion_pytorch_model.safetensors");

std::string vae_path = path_join(file_path, "vae/diffusion_pytorch_model.safetensors");

std::string clip_path = path_join(file_path, "text_encoder/model.safetensors");

std::string clip_g_path = path_join(file_path, "text_encoder_2/model.safetensors");

if (!parse_file(unet_path, prefix + "unet.")) {

return false;

}

if (file_exists(vae_path) && !parse_file(vae_path, prefix + "vae.")) {

return false;

}

if (file_exists(clip_path) && !parse_file(clip_path, prefix + "te.")) {

return false;

}

if (file_exists(clip_g_path) && !parse_file(clip_g_path, prefix + "te.1.")) {

return false;

}

return true;

TensorStorage token_embedding_weight, input_block_weight, context_ebedding_weight;

bool has_multiple_encoders = false;

bool is_unet = false;

bool is_xl = false;

bool is_flux = false;

bool is_flux2 = false;

bool has_single_block_47 = false;

bool is_wan = false;

bool is_s2v = false;

int64_t patch_embedding_channels = 0;

bool has_img_emb = false;

bool has_middle_block_1 = false;

bool has_output_block_311 = false;

bool has_output_block_71 = false;

bool has_attn_1024 = false;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos ||

tensor_storage.name.find("model.diffusion_model.single_transformer_blocks.") != std::string::npos) {

is_flux = true;

}

if (tensor_storage.name.find("model.diffusion_model.net.lq_proj.latent_proj.0.weight") != std::string::npos) {

return VERSION_PID;

}

if (tensor_storage.name.find("embed_image_indicator.weight") != std::string::npos) {

return VERSION_IDEOGRAM4;

}

if (tensor_storage.name.find("model.diffusion_model.txtfusion.projector.weight") != std::string::npos ||

tensor_storage.name.find("model.diffusion_model.text_fusion.projector.weight") != std::string::npos) {

return VERSION_KREA2;

}

if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {

return VERSION_CHROMA_RADIANCE;

}

if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) {

return VERSION_SD3;

}

if (tensor_storage.name.find("model.x_embedder.proj1.weight") != std::string::npos &&

tensor_storage_map.find("model.language_model.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {

return VERSION_HIDREAM_O1;

}

if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.attn.norm_added_q.weight") != std::string::npos &&

tensor_storage_map.find("model.diffusion_model.transformer_blocks.0.img_mlp.w1.weight") != tensor_storage_map.end()) {

return VERSION_LENS;

}

if (tensor_storage.name.find("net.img_embedder.proj1.weight") != std::string::npos) {

return VERSION_MINIT2I;

}

if (tensor_storage.name.find("language_model.model.layers.0.self_attn.q_proj_mot_gen.weight") != std::string::npos) {

return VERSION_SENSENOVA_U1_5;

}

if (tensor_storage.name == "model.diffusion_model.txt_in.text_norm.weight") {

return VERSION_QWEN_IMAGE_2_1;

}

if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.img_mod.1.weight") != std::string::npos) {

auto img_in = tensor_storage_map.find("model.diffusion_model.img_in.weight");

if (img_in != tensor_storage_map.end() && img_in->second.ne[0] == 128) {

return VERSION_MAGE_FLOW;

}

if (tensor_storage_map.find("model.diffusion_model.time_text_embed.addition_t_embedding.weight") != tensor_storage_map.end()) {

return VERSION_QWEN_IMAGE_LAYERED;

}

return VERSION_QWEN_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.txt_in.individual_token_refiner.blocks.0.adaLN_modulation.1.weight") != std::string::npos) {

return VERSION_HUNYUAN_VIDEO;

}

if (tensor_storage.name.find("llm_adapter.blocks.0.cross_attn.q_proj.weight") != std::string::npos) {

return VERSION_ANIMA;

}

if (tensor_storage.name.find("model.diffusion_model.double_stream_modulation_img.lin.weight") != std::string::npos) {

is_flux2 = true;

}

if (tensor_storage.name.find("dual_time_embed.semantic_embedder.linear_1.weight") != std::string::npos) {

return VERSION_SEFI_IMAGE;

}

if (tensor_storage.name.find("single_blocks.47.linear1.weight") != std::string::npos) {

has_single_block_47 = true;

}

if (tensor_storage.name.find("model.diffusion_model.double_blocks.0.img_mlp.gate_proj.weight") != std::string::npos) {

return VERSION_OVIS_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.sigvq_embedder.1.weight") != std::string::npos) {

return VERSION_LLADA_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {

if (tensor_storage_map.find("text_encoders.llm.connector.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {

return VERSION_MING_IMAGE;

}

return VERSION_Z_IMAGE;

}

if (tensor_storage.name.find("double_stream_layers.0.img_instruct_attn.processor.img_to_q.weight") != std::string::npos) {

return VERSION_BOOGU_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.layers.0.adaLN_sa_ln.weight") != std::string::npos) {

return VERSION_ERNIE_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.t_block.1.weight") != std::string::npos &&

tensor_storage_map.find("model.diffusion_model.x_embedder.proj.weight") != tensor_storage_map.end() &&

tensor_storage_map.find("model.diffusion_model.audio_patchify_proj.weight") == tensor_storage_map.end()) {

return VERSION_PIXART;

}

if (tensor_storage.name.find("model.diffusion_model.adaln_single.emb.timestep_embedder.linear_1.bias") != std::string::npos) {

// PixArt shares this timestep embedding with LTX-AV.

if (tensor_storage_map.find("model.diffusion_model.pos_embed.proj.weight") != tensor_storage_map.end() &&

tensor_storage_map.find("model.diffusion_model.adaln_single.linear.weight") != tensor_storage_map.end() &&

tensor_storage_map.find("model.diffusion_model.audio_patchify_proj.weight") == tensor_storage_map.end()) {

return VERSION_PIXART;

}

return VERSION_LTXAV;

}

if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos &&

tensor_storage_map.find("model.diffusion_model.audio_patch_proj.weight") != tensor_storage_map.end()) {

return VERSION_MINIMAX_H3;

}

if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) {

is_wan = true;

}

if (tensor_storage.name.find("casual_audio_encoder.weights") != std::string::npos ||

tensor_storage.name.find("audio_injector.injector.0.q.weight") != std::string::npos) {

// S2V and T2V-14B share patch_embedding shapes.

is_s2v = true;

}

if (tensor_storage.name.find("model.diffusion_model.patch_embedder.weight") != std::string::npos) {

return VERSION_LINGBOT_VIDEO;

}

if (tensor_storage.name.find("model.diffusion_model.patch_embedding.weight") != std::string::npos) {

patch_embedding_channels = tensor_storage.ne[3];

}

if (tensor_storage.name.find("model.diffusion_model.img_emb") != std::string::npos) {

has_img_emb = true;

}

if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos ||

tensor_storage.name.find("unet.down_blocks.") != std::string::npos) {

is_unet = true;

if (has_multiple_encoders) {

is_xl = true;

}

}

if (tensor_storage.name.find("conditioner.embedders.1") != std::string::npos ||

tensor_storage.name.find("cond_stage_model.1") != std::string::npos ||

tensor_storage.name.find("te.1") != std::string::npos) {

has_multiple_encoders = true;

if (is_unet) {

is_xl = true;

}

}

if (tensor_storage.name.find("model.diffusion_model.input_blocks.8.0.time_mixer.mix_factor") != std::string::npos) {

return VERSION_SVD;

}

if (tensor_storage.name.find("model.diffusion_model.middle_block.1.") != std::string::npos ||

tensor_storage.name.find("unet.mid_block.resnets.1.") != std::string::npos) {

has_middle_block_1 = true;

}

if (tensor_storage.name.find("model.diffusion_model.output_blocks.3.1.transformer_blocks.1") != std::string::npos ||

tensor_storage.name.find("unet.up_blocks.1.attentions.0.transformer_blocks.1") != std::string::npos) {

has_output_block_311 = true;

}

if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1") != std::string::npos ||

tensor_storage.name.find("unet.up_blocks.2.attentions.1") != std::string::npos) {

has_output_block_71 = true;

if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k.weight") != std::string::npos) {

if (tensor_storage.ne[0] == 1024)

has_attn_1024 = true;

}

}

if (tensor_storage.name == "cond_stage_model.transformer.text_model.embeddings.token_embedding.weight" ||

tensor_storage.name == "cond_stage_model.model.token_embedding.weight" ||

tensor_storage.name == "text_model.embeddings.token_embedding.weight" ||

tensor_storage.name == "te.text_model.embeddings.token_embedding.weight" ||

tensor_storage.name == "conditioner.embedders.0.model.token_embedding.weight" ||

tensor_storage.name == "conditioner.embedders.0.transformer.text_model.embeddings.token_embedding.weight") {

token_embedding_weight = tensor_storage;

// break;

}

if (tensor_storage.name == "model.diffusion_model.input_blocks.0.0.weight" ||

tensor_storage.name == "model.diffusion_model.img_in.weight" ||

tensor_storage.name == "unet.conv_in.weight") {

input_block_weight = tensor_storage;

}

if (tensor_storage.name == "model.diffusion_model.txt_in.weight" || tensor_storage.name == "model.diffusion_model.context_embedder.weight") {

context_ebedding_weight = tensor_storage;

}

}

if (is_wan) {

LOG_VERBOSE("patch_embedding_channels %d", patch_embedding_channels);

if (is_s2v) {

return VERSION_WAN2_2_S2V;

}

if (patch_embedding_channels == 184320 && !has_img_emb) {

return VERSION_WAN2_2_I2V;

}

if (patch_embedding_channels == 147456 && !has_img_emb) {

return VERSION_WAN2_2_TI2V;

}

return VERSION_WAN2;

}

bool is_inpaint = input_block_weight.ne[2] == 9;

bool is_ip2p = input_block_weight.ne[2] == 8;

if (is_xl) {

if (is_inpaint) {

return VERSION_SDXL_INPAINT;

}

if (is_ip2p) {

return VERSION_SDXL_PIX2PIX;

}

if (!has_middle_block_1) {

if (!has_output_block_311) {

return VERSION_SDXL_VEGA;

}

return VERSION_SDXL_SSD1B;

}

return VERSION_SDXL;

}

if (is_flux && !is_flux2) {

if (context_ebedding_weight.ne[0] == 3584) {

return VERSION_LONGCAT;

} else {

if (input_block_weight.ne[0] == 384) {

return VERSION_FLUX_FILL;

}

if (input_block_weight.ne[0] == 128) {

return VERSION_FLUX_CONTROLS;

}

if (input_block_weight.ne[0] == 196) {

return VERSION_FLEX_2;

}

return VERSION_FLUX;

}

}

if (is_flux2) {

if (has_single_block_47) {

return VERSION_FLUX2;

}

return VERSION_FLUX2_KLEIN;

}

if (token_embedding_weight.ne[0] == 768) {

if (is_inpaint) {

return VERSION_SD1_INPAINT;

}

if (is_ip2p) {

return VERSION_SD1_PIX2PIX;

}

if (!has_middle_block_1) {

if (!has_output_block_71) {

return VERSION_SDXS_512_DS;

}

return VERSION_SD1_TINY_UNET;

}

return VERSION_SD1;

} else if (token_embedding_weight.ne[0] == 1024) {

if (is_inpaint) {

return VERSION_SD2_INPAINT;

}

if (!has_middle_block_1) {

return has_attn_1024 ? VERSION_SDXS_09 : VERSION_SD2_TINY_UNET;

}

return VERSION_SD2;

}

return VERSION_COUNT;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if ((tensor_storage.name.find("text_encoders") == std::string::npos &&

tensor_storage.name.find("cond_stage_model") == std::string::npos &&

tensor_storage.name.find("te.text_model.") == std::string::npos &&

tensor_storage.name.find("conditioner") == std::string::npos)) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (tensor_storage.name.find("model.diffusion_model.") == std::string::npos && tensor_storage.name.find("unet.") == std::string::npos) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (tensor_storage.name.find("vae.") == std::string::npos &&

tensor_storage.name.find("first_stage_model") == std::string::npos) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

TensorTypeRules result;

for (const auto& item : split_string(tensor_type_rules, ',')) {

if (item.size() == 0)

continue;

std::string::size_type pos = item.find('=');

if (pos == std::string::npos) {

LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str());

continue;

}

std::string tensor_pattern = item.substr(0, pos);

std::string type_name = item.substr(pos + 1);

ggml_type tensor_type = GGML_TYPE_COUNT;

if (type_name == "f32") {

tensor_type = GGML_TYPE_F32;

} else {

for (size_t i = 0; i < GGML_TYPE_COUNT; i++) {

auto trait = ggml_get_type_traits((ggml_type)i);

if (trait->to_float && trait->type_size && type_name == trait->type_name) {

tensor_type = (ggml_type)i;

}

}

}

if (tensor_type != GGML_TYPE_COUNT) {

result.emplace_back(tensor_pattern, tensor_type);

} else {

LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str());

}

}

return result;

wtype_override_ = wtype;

tensor_type_rules_ = tensor_type_rules;

auto map_rules = parse_tensor_type_rules(tensor_type_rules);

for (auto& [name, tensor_storage] : tensor_storage_map) {

tensor_storage.expected_type = GGML_TYPE_COUNT;

ggml_type dst_type = wtype;

for (const auto& tensor_type_rule : map_rules) {

std::regex pattern(tensor_type_rule.first);

if (std::regex_search(name, pattern)) {

dst_type = tensor_type_rule.second;

break;

}

}

if (dst_type == GGML_TYPE_COUNT) {

continue;

}

if (!tensor_should_be_converted(tensor_storage, dst_type)) {

continue;

}

tensor_storage.expected_type = dst_type;

}

invalidate_file_data();

++revision_;

if (model_files_processed) {

return;

}

std::vector<TensorStorage> processed_tensor_storages;

for (const auto& [name, tensor_storage] : tensor_storage_map) {

processed_tensor_storages.push_back(tensor_storage);

}

for (size_t file_index = 0; file_index < file_paths_.size(); file_index++) {

std::string file_path = file_paths_[file_index];

std::vector<TensorStorage> file_tensors;

for (const auto& ts : processed_tensor_storages) {

if (ts.file_index == file_index) {

file_tensors.push_back(ts);

}

}

if (file_tensors.empty()) {

continue;

}

bool is_zip = false;

for (auto const& ts : file_tensors) {

if (ts.index_in_zip >= 0) {

is_zip = true;

break;

}

}

ModelFileData fdata = {};

fdata.path = file_path;

fdata.is_zip = is_zip;

fdata.tensors = std::move(file_tensors);

if (enable_mmap && !is_zip) {

LOG_VERBOSE("using mmap for I/O");

std::unique_ptr<MmapWrapper> mmapped = MmapWrapper::create(file_path, writable_mmap);

if (mmapped) {

uint8_t* mmap_data = static_cast<uint8_t*>(mmapped->writable_data());

ggml_backend_buffer_t buf_mmap = ggml_backend_cpu_buffer_from_ptr(mmap_data, mmapped->size());

if (buf_mmap) {

LOG_INFO("using mmap for '%s'", file_path.c_str());

fdata.mmbuffer = std::shared_ptr<struct ggml_backend_buffer>(buf_mmap, ggml_backend_buffer_free);

} else {

LOG_WARN("mmap: failed to create backend buffer for file %s", fdata.path.c_str());

}

fdata.mmapped = std::shared_ptr<MmapWrapper>(std::move(mmapped));

} else {

LOG_WARN("failed to memory-map '%s' (falling back to read())", file_path.c_str());

}

}

file_data.push_back(std::move(fdata));

}

model_files_processed = true;

std::set<std::string> ignore_tensors,

bool writable_mmap,

ggml_backend_dev_t device) {

std::set<std::string> names;

for (const auto& entry : tensors) {

names.insert(entry.first);

}

if (!validate_sources(&names)) {

return {};

}

process_model_files(true, writable_mmap);

std::vector<MmapTensorStore> result;

uint64_t mapped_bytes = 0;

size_t mapped_tensors = 0;

LOG_VERBOSE("memory-mapping tensors...");

int64_t t_start = ggml_time_ms();

for (auto& fdata : file_data) {

if (!fdata.mmbuffer)

continue;

// Wrapped on first use: a device buffer makes the whole file resident on that device.

std::shared_ptr<struct ggml_backend_buffer> file_buffer = device == nullptr ? fdata.mmbuffer : nullptr;

bool file_unmappable = false;

auto buffer_for_file = [&]() -> ggml_backend_buffer_t {

if (file_buffer || file_unmappable) {

return file_buffer.get();

}

auto cached = fdata.device_mmbuffers.find(device);

if (cached != fdata.device_mmbuffers.end()) {

file_buffer = cached->second;

return file_buffer.get();

}

size_t max_tensor_size = 0;

for (const auto& ts : fdata.tensors) {

max_tensor_size = std::max(max_tensor_size, static_cast<size_t>(ts.nbytes()));

}

ggml_backend_buffer_t buf = sd_backend_dev_buffer_from_host_ptr(device,

fdata.mmapped->writable_data(),

fdata.mmapped->size(),

max_tensor_size);

if (buf == nullptr) {

LOG_WARN("mmap: %s cannot map '%s', loading it instead",

ggml_backend_dev_name(device), fdata.path.c_str());

file_unmappable = true;

return nullptr;

}

LOG_INFO("mmap: mapped '%s' for %s", fdata.path.c_str(), ggml_backend_dev_name(device));

file_buffer = std::shared_ptr<struct ggml_backend_buffer>(buf, ggml_backend_buffer_free);

fdata.device_mmbuffers[device] = file_buffer;

return file_buffer.get();

};

const std::vector<TensorStorage>& file_tensors = fdata.tensors;

size_t file_mapped_bytes = 0;

size_t file_mapped_tensors = 0;

for (const auto& tensor_storage : file_tensors) {

const std::string& name = tensor_storage.name;

bool is_ignored = false;

for (const auto& ignore_prefix : ignore_tensors) {

if (starts_with(name, ignore_prefix)) {

is_ignored = true;

break;

}

}

if (is_ignored)

continue;

auto it = tensors.find(name);

if (it == tensors.end())

continue;

ggml_tensor* dst_tensor = it->second;

if (dst_tensor == nullptr)

continue;

if (tensor_storage.is_f64 ||

tensor_storage.is_i64 ||

#ifdef SD_USE_UPSTREAM_GGML

tensor_storage.is_f8_e4m3 ||

tensor_storage.is_f8_e5m2 ||

#endif

tensor_storage.type != dst_tensor->type) {

continue;

}

size_t tensor_size = tensor_storage.nbytes();

size_t tensor_offset = tensor_storage.offset;

if (tensor_storage.ne[0] != dst_tensor->ne[0] ||

tensor_storage.ne[1] != dst_tensor->ne[1] ||

tensor_storage.ne[2] != dst_tensor->ne[2] ||

tensor_storage.ne[3] != dst_tensor->ne[3] ||

tensor_size != ggml_nbytes(dst_tensor)) {

// let load_tensors worry about this

continue;

}