GGML_ASSERT(alignment != 0 && (alignment & (alignment - 1)) == 0);

size_t align = (alignment - ((reinterpret_cast<uintptr_t>(buffer) + offset) % alignment)) % alignment;

return offset + align;

const std::vector<ModelManager::LoraSpec>& rhs) {

if (lhs.size() != rhs.size()) {

return false;

}

for (size_t i = 0; i < lhs.size(); ++i) {

if (lhs[i].path != rhs[i].path ||

lhs[i].multiplier != rhs[i].multiplier ||

lhs[i].is_high_noise != rhs[i].is_high_noise ||

lhs[i].tensor_name_prefix_filter != rhs[i].tensor_name_prefix_filter ||

lhs[i].required != rhs[i].required ||

lhs[i].file_id != rhs[i].file_id || lhs[i].file_revision != rhs[i].file_revision) {

return false;

}

}

return true;

if (backend == nullptr) {

return false;

}

if (sd_backend_is_cpu(backend)) {

return true;

}

ggml_backend_dev_t dev = ggml_backend_get_device(backend);

if (dev == nullptr) {

return false;

}

ggml_backend_dev_props props;

ggml_backend_dev_get_props(dev, &props);

return props.caps.buffer_from_host_ptr;

ggml_tensor* weight,

enum ggml_op op,

ggml_backend_buffer_type_t buft) {

if (op == GGML_OP_NONE) {

return true;

}

if (device == nullptr || weight == nullptr || buft == nullptr || weight->buffer != nullptr) {

return false;

}

ggml_init_params params;

params.mem_size = ggml_tensor_overhead() * 2;

params.mem_buffer = nullptr;

params.no_alloc = true;

ggml_context* ctx = ggml_init(params);

if (ctx == nullptr) {

return false;

}

ggml_tensor* op_tensor = nullptr;

if (op == GGML_OP_GET_ROWS) {

ggml_tensor* indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);

op_tensor = ggml_get_rows(ctx, weight, indices);

} else if (op == GGML_OP_CUSTOM) {

op_tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);

op_tensor->op = op;

op_tensor->src[0] = weight;

}

if (op_tensor == nullptr) {

ggml_free(ctx);

return false;

}

weight->buffer = ggml_backend_buft_alloc_buffer(buft, 0);

if (weight->buffer == nullptr) {

ggml_free(ctx);

return false;

}

bool supported = ggml_backend_dev_supports_op(device, op_tensor);

ggml_backend_buffer_free(weight->buffer);

weight->buffer = nullptr;

ggml_free(ctx);

return supported;

ModelLoader candidate = model_loader_;

std::vector<LoraSpec> resolved;

std::set<ModelLoader::FileId> sources;

for (auto spec : loras) {

const std::string prefix = spec.is_high_noise ? "lora.model.high_noise_" : "lora.";

if (!candidate.add_file(spec.path, prefix, &spec.file_id, false, ModelLoader::FileScope::Isolated)) {

if (spec.required)

return false;

LOG_WARN("cannot register LoRA source '%s'", spec.path.c_str());

continue;

}

spec.file_revision = candidate.file_revision(spec.file_id);

sources.insert(spec.file_id);

resolved.push_back(std::move(spec));

}

for (auto id : lora_sources_) {

if (sources.count(id) == 0)

candidate.del_file(id);

}

if (!set_loader(std::move(candidate)))

return false;

lora_sources_ = std::move(sources);

loras = std::move(resolved);

return true;

if (std::any_of(loras.begin(), loras.end(), [](const LoraSpec& spec) { return spec.file_id == 0; }) &&

!prepare_lora_sources(loras))

return false;

for (auto& spec : loras) {

spec.file_revision = model_loader_.file_revision(spec.file_id);

if (spec.file_revision == 0)

return false;

}

if (lora_version_ == version && lora_specs_equal(loras_, loras))

return true;

if (!workspace_reclaimers_.empty() || std::any_of(tensor_states_.begin(), tensor_states_.end(), [](const auto& state) {

return state->pin_count != 0;

})) {

LOG_ERROR("cannot change LoRA configuration during execution");

return false;

}

loras_ = std::move(loras);

lora_version_ = version;

current_lora_epoch_++;

reset_lora_applied_params();

return true;

std::set<std::string> names;

for (const auto& state : tensor_states_) {

if (state != nullptr && state->component != ModelComponent::LoRA) {

names.insert(state->name);

}

}

return names;

size_t size = 0;

std::unordered_set<ggml_tensor*> seen;

for (const auto& pair : tensors) {

ggml_tensor* tensor = pair.second;

if (tensor == nullptr || seen.find(tensor) != seen.end()) {

continue;

}

seen.insert(tensor);

size += ggml_nbytes(tensor);

}

return size;

if (compute_backend == nullptr) {

return;

}

if (split_buft == nullptr) {

split_buffer_types_.erase(compute_backend);

return;

}

split_buffer_types_[compute_backend] = split_buft;

split_buffer_devices_[split_buft] = device_limits;

return tensor != nullptr &&

tensor->view_src == nullptr &&

ggml_is_contiguous(tensor) &&

ggml_n_dims(tensor) == 2 &&

tensor->ne[0] >= 256 &&

tensor->ne[1] >= 256;

if (!tensor_shape_supports_split_buffer(state.tensor)) {

return nullptr;

}

return state.split_buffer_type;

std::map<std::string, ggml_tensor*> tensors,

ResidencyMode residency_mode,

ggml_backend_t compute_backend,

ggml_backend_t params_backend,

size_t* registered_tensor_size,

bool allow_split_buffer,

bool params_follow_compute_backend,

const std::map<ggml_tensor*, enum ggml_op>* tensor_ops,

ModelLoader::FileId source_file,

SDVersion source_version) {

if (component == ModelComponent::Count) {

LOG_ERROR("model manager tensor component is invalid");

return false;

}

if (registered_tensor_size != nullptr) {

*registered_tensor_size += estimate_tensors_size(tensors);

}

const auto scoped_sources = source_file != 0 ? model_loader_.file_tensors(source_file, source_version) : String2TensorStorage{};

const auto& sources = source_file != 0 ? scoped_sources : model_loader_.get_tensor_storage_map();

std::unordered_set<ggml_tensor*> new_tensors;

std::vector<std::unique_ptr<TensorState>> new_states;

new_states.reserve(tensors.size());

for (const auto& pair : tensors) {

const std::string& name = pair.first;

ggml_tensor* tensor = pair.second;

if (tensor == nullptr) {

continue;

}

if (tensor_states_by_tensor_.count(tensor) != 0 || !new_tensors.insert(tensor).second) {

LOG_ERROR("model manager tensor name '%s' is already registered", name.c_str());

return false;

}

ggml_set_name(tensor, name.c_str());

auto state = std::make_unique<TensorState>();

state->name = name;

state->tensor = tensor;

state->component = component;

state->source_file = source_file;

state->source_version = source_version;

state->sources = find_tensor_sources(*state, sources);

state->residency_mode = residency_mode;

state->compute_backend = compute_backend;

state->params_backend = params_backend;

auto split_buffer = split_buffer_types_.find(compute_backend);

if (allow_split_buffer && split_buffer != split_buffer_types_.end()) {

state->split_buffer_type = split_buffer->second;

}

state->params_follow_compute_backend = params_follow_compute_backend;

if (tensor_ops != nullptr) {

auto op_it = tensor_ops->find(tensor);

if (op_it != tensor_ops->end()) {

state->usage_op = op_it->second;

}

}

new_states.push_back(std::move(state));

}

resolved_tensor_states_.clear();

for (auto& state : new_states) {

TensorState* registered_state = state.get();

tensor_states_by_tensor_[registered_state->tensor] = registered_state;

tensor_states_.push_back(std::move(state));

}

return true;

std::unordered_set<TensorState*> states;

for (auto& state : tensor_states_) {

if (state->component == component)

states.insert(state.get());

}

return unregister_tensor_states(states, registered_tensor_size);

std::unordered_set<TensorState*> states;

for (auto tensor : tensors) {

auto found = tensor_states_by_tensor_.find(tensor);

if (found != tensor_states_by_tensor_.end())

states.insert(found->second);

}

return unregister_tensor_states(states, nullptr);

size_t* registered_tensor_size) {

size_t released_size = 0;

for (auto& state : tensor_states_) {

if (state == nullptr || target_states.count(state.get()) == 0) {

continue;

}

if (state->pin_count > 0) {

LOG_ERROR("model manager cannot unregister active %s tensor '%s'",

model_component_name(state->component),

state->name.c_str());

return false;

}

if (state->tensor != nullptr) {

released_size += ggml_nbytes(state->tensor);

}

}

if (target_states.empty()) {

return true;

}

clear_all_prefetched_params();

release_compute_staging_blocks(false, &target_states);

std::vector<ParamsStorageBlock*> storage_blocks_to_release;

std::unordered_set<TensorState*> affected_storage_states;

for (const auto& block : params_storage_blocks_) {

if (block == nullptr) {

continue;

}

bool has_target_state = false;

for (TensorState* state : block->states) {

if (state != nullptr && target_states.count(state) > 0) {

has_target_state = true;

break;

}

}

if (!has_target_state) {

continue;

}

storage_blocks_to_release.push_back(block.get());

for (TensorState* state : block->states) {

if (state != nullptr) {

affected_storage_states.insert(state);

}

}

}

for (TensorState* state : affected_storage_states) {

if (state == nullptr) {

continue;

}

if (state->pin_count > 0 || state->staged_to_compute_backend) {

LOG_ERROR("model manager cannot unregister %s while tensor '%s' is active",

model_component_name(state->component),

state->name.c_str());

return false;

}

}

for (ParamsStorageBlock* block : storage_blocks_to_release) {

if (block != nullptr) {

free_params_storage_block(*block);

erase_params_storage_block(block);

}

}

resolved_tensor_states_.clear();

for (auto it = tensor_states_by_tensor_.begin(); it != tensor_states_by_tensor_.end();) {

if (target_states.count(it->second) > 0) {

it = tensor_states_by_tensor_.erase(it);

} else {

++it;

}

}

tensor_states_.erase(std::remove_if(tensor_states_.begin(),

tensor_states_.end(),

[&](const std::unique_ptr<TensorState>& s) {

return s == nullptr || target_states.count(s.get()) > 0;

}),

tensor_states_.end());

if (registered_tensor_size != nullptr) {

if (released_size > *registered_tensor_size) {

*registered_tensor_size = 0;

} else {

*registered_tensor_size -= released_size;

}

}

return true;

std::vector<TensorState*> all_states;

all_states.reserve(tensor_states_.size());

for (const auto& s : tensor_states_) {

if (s != nullptr) {

all_states.push_back(s.get());

}

}

return load_tensors_to_params_backend(all_states);

bool ok = true;

for (const auto& state : tensor_states_) {

if (state == nullptr) {

ok = false;

continue;

}

bool state_ok = validate_tensor(*state);

if (state_ok) {

state->metadata_validated = true;

}

ok = state_ok && ok;

}

return ok;

std::vector<TensorState*> need_load;

need_load.reserve(states.size());

for (TensorState* state : states) {

if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) {

continue;

}

if (!state->metadata_validated) {

if (!validate_tensor(*state)) {

return false;

}

state->metadata_validated = true;

}

if (!state->loaded_to_params_backend) {

need_load.push_back(state);

}

}

if (need_load.empty()) {

return true;

}

std::vector<ParamsStorageBlock*> created_storage_blocks;

if (!mmap_params(need_load, created_storage_blocks)) {

for (ParamsStorageBlock* block : created_storage_blocks) {

if (block != nullptr) {

free_params_storage_block(*block);

erase_params_storage_block(block);

}

}

return false;

}

std::vector<TensorState*> need_alloc;

need_alloc.reserve(need_load.size());

for (TensorState* state : need_load) {

if (state->tensor != nullptr && state->tensor->data == nullptr && state->tensor->view_src == nullptr) {

need_alloc.push_back(state);

}

}

if (!alloc_params_buffers(need_alloc, created_storage_blocks) ||

!load_tensors(need_load)) {

for (ParamsStorageBlock* block : created_storage_blocks) {

if (block != nullptr) {

free_params_storage_block(*block);

erase_params_storage_block(block);

}

}

return false;

}

struct PrepareStats {

size_t bytes = 0;

size_t tensors = 0;

size_t blocks = 0;

};

std::map<ggml_backend_buffer_type_t, PrepareStats> prepared;

for (ParamsStorageBlock* block : created_storage_blocks) {

if (block != nullptr && block->buffer != nullptr) {

auto& stats = prepared[ggml_backend_buffer_get_type(block->buffer)];

stats.bytes += ggml_backend_buffer_get_size(block->buffer);

stats.tensors += block->states.size();

++stats.blocks;

}

}

for (const auto& entry : prepared) {

LOG_VERBOSE("model manager prepared params backend buffers (%6.2f MB, %zu tensors, %zu blocks, %s) on %s",

entry.second.bytes / (1024.f * 1024.f),

entry.second.tensors, entry.second.blocks,

ggml_backend_buft_is_host(entry.first) ? "RAM" : "VRAM",

ggml_backend_buft_name(entry.first));

}

return true;

std::map<std::pair<ggml_backend_t, ggml_backend_buffer_type_t>, std::vector<TensorState*>> states_by_staging_target;

for (TensorState* state : states) {

if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) {

continue;

}

if (state->compute_backend == nullptr) {

LOG_ERROR("model manager compute backend is null for tensor '%s'", state->name.c_str());

return false;

}

if (state->params_backend == nullptr) {

LOG_ERROR("model manager params backend is null for tensor '%s'", state->name.c_str());

return false;

}

// Custom CPU operators must retain host weights even when the runner uses a GPU.

if (state->usage_op == GGML_OP_CUSTOM ||

state->compute_backend == state->params_backend || state->staged_to_compute_backend) {

continue;

}

if (!state->loaded_to_params_backend || state->tensor == nullptr || state->tensor->data == nullptr) {

LOG_ERROR("model manager tensor '%s' is not loaded to params backend", state->name.c_str());

return false;

}

ggml_backend_buffer_type_t staging_buft = split_buffer_type_for(*state);

if (staging_buft == nullptr) {

staging_buft = ggml_backend_get_default_buffer_type(state->compute_backend);

}

states_by_staging_target[{state->compute_backend, staging_buft}].push_back(state);

}

for (const auto& pair : states_by_staging_target) {

ggml_backend_t compute_backend = pair.first.first;

ggml_backend_buffer_type_t staging_buft = pair.first.second;

const std::vector<TensorState*>& target_states = pair.second;

if (target_states.empty()) {

continue;

}

const size_t alignment = ggml_backend_buft_get_alignment(staging_buft);

size_t backend_limit = ggml_backend_buft_get_max_size(staging_buft);

if (!ggml_backend_buft_is_host(staging_buft) &&

(backend_limit == 0 || backend_limit > MAX_RESIDENCY_BLOCK_BYTES)) {

backend_limit = MAX_RESIDENCY_BLOCK_BYTES;

}

const int64_t t0 = ggml_time_ms();

size_t staged_bytes = 0;

size_t staged_blocks = 0;

auto stage_chunk = [&](const std::vector<TensorState*>& chunk) -> bool {

if (chunk.empty()) {

return true;

}

ggml_init_params init_params;

init_params.mem_size = std::max<size_t>(1, chunk.size()) * ggml_tensor_overhead();

init_params.mem_buffer = nullptr;

init_params.no_alloc = true;

ggml_context* staging_ctx = ggml_init(init_params);

GGML_ASSERT(staging_ctx != nullptr);

std::vector<std::pair<TensorState*, ggml_tensor*>> staged_tensors;

staged_tensors.reserve(chunk.size());

for (TensorState* state : chunk) {

ggml_tensor* staging_tensor = ggml_dup_tensor(staging_ctx, state->tensor);

ggml_set_name(staging_tensor, state->tensor->name);

staged_tensors.push_back({state, staging_tensor});

}

ggml_backend_buffer_t compute_buffer =

ggml_backend_alloc_ctx_tensors_from_buft(staging_ctx, staging_buft);

if (compute_buffer == nullptr) {

LOG_ERROR("model manager alloc compute params backend buffer failed, num_tensors = %zu",

staged_tensors.size());

ggml_free(staging_ctx);

return false;

}

ggml_backend_buffer_set_usage(compute_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);

for (auto& staged_tensor : staged_tensors) {

TensorState* state = staged_tensor.first;

ggml_tensor* managed_tensor = state->tensor;

ggml_tensor* staging_tensor = staged_tensor.second;

ggml_backend_tensor_copy(managed_tensor, staging_tensor);

std::swap(managed_tensor->buffer, staging_tensor->buffer);

std::swap(managed_tensor->data, staging_tensor->data);

std::swap(managed_tensor->extra, staging_tensor->extra);

state->staged_to_compute_backend = true;

}

ggml_backend_synchronize(compute_backend);

auto block = std::make_unique<ComputeStagingBlock>();

block->compute_backend = compute_backend;

block->buffer = compute_buffer;

block->staging_ctx = staging_ctx;

block->staged_tensors = std::move(staged_tensors);

staged_bytes += ggml_backend_buffer_get_size(compute_buffer);

++staged_blocks;

compute_staging_blocks_.push_back(std::move(block));

return true;

};

std::vector<TensorState*> chunk;

size_t chunk_size = 0;

for (TensorState* state : target_states) {

const size_t tensor_size = GGML_PAD(

ggml_backend_buft_get_alloc_size(staging_buft, state->tensor), alignment);

if (!chunk.empty() && backend_limit > 0 &&

tensor_size > backend_limit - std::min(chunk_size, backend_limit)) {

if (!stage_chunk(chunk)) {

return false;

}

chunk.clear();

chunk_size = 0;

}

chunk.push_back(state);

chunk_size = tensor_size > SIZE_MAX - chunk_size ? SIZE_MAX : chunk_size + tensor_size;

}

if (!stage_chunk(chunk)) {

return false;

}

LOG_VERBOSE("model manager staged compute params (%6.2f MB, %zu tensors, %zu blocks) to %s, taking %.2fs",

staged_bytes / (1024.f * 1024.f),

target_states.size(),

staged_blocks,

ggml_backend_name(compute_backend),

(ggml_time_ms() - t0) / 1000.f);

}

return true;

if (loras_.empty() || applying_loras_)

return true;

applying_loras_ = true;

struct ApplyGuard {

bool& active;

~ApplyGuard() { active = false; }

} guard{applying_loras_};

struct LoraApplyGroup {

std::map<std::string, ggml_tensor*> model_tensors;

std::vector<TensorState*> states;

};

using ApplyTarget = std::tuple<ggml_backend_t, ggml_backend_t, ResidencyMode>;

std::map<ApplyTarget, LoraApplyGroup> groups;

for (TensorState* state : states) {

if (state == nullptr || state->tensor == nullptr || state->component == ModelComponent::LoRA ||

state->component == ModelComponent::LatentUpsampler || should_ignore(*state) || is_optional_missing_tensor(state->name)) {

continue;

}

if (state->applied_lora_epoch == current_lora_epoch_) {

continue;

}

if (state->compute_backend == nullptr) {

LOG_ERROR("model manager compute backend is null for lora target tensor '%s'", state->name.c_str());

return false;

}

if (state->tensor->buffer != nullptr &&

ggml_backend_buffer_get_type(state->tensor->buffer) == split_buffer_type_for(*state)) {

if (!warned_split_lora_skip_) {

LOG_WARN(

"model manager skipping direct lora application to row-split tensors "

"(use --lora-apply-mode at_runtime with row split)");

warned_split_lora_skip_ = true;

}

state->applied_lora_epoch = current_lora_epoch_;

continue;

}

if (state->tensor->data == nullptr) {

LOG_ERROR("model manager lora target tensor '%s' is not prepared", state->name.c_str());

return false;

}

LoraApplyGroup& group = groups[{state->compute_backend, state->params_backend, state->residency_mode}];

group.model_tensors[state->name] = state->tensor;

group.states.push_back(state);

}

if (groups.empty()) {

return true;

}

std::set<std::string> all_tensor_names = tensor_names();

for (auto& group_pair : groups) {

ggml_backend_t compute_backend = std::get<0>(group_pair.first);

LoraApplyGroup& group = group_pair.second;

for (const LoraSpec& lora_spec : loras_) {

if (group.model_tensors.empty()) {

continue;

}

std::string id = lora_id(lora_spec);

const auto* target = group.states.front();

// The temporary runner is destroyed before this manager call returns.

auto borrowed_manager = std::shared_ptr<ModelManager>(this, [](ModelManager*) {});

auto lora = std::make_shared<LoraModel>(id, compute_backend, target->params_backend,

borrowed_manager, lora_spec.file_id, lora_version_,

target->residency_mode);

LoraModel::filter_t lora_tensor_filter = nullptr;

if (!lora_spec.tensor_name_prefix_filter.empty()) {

lora_tensor_filter = [&](const std::string& tensor_name) {

return starts_with(tensor_name, lora_spec.tensor_name_prefix_filter);

};

}

if (!lora->init_params(n_threads_, lora_tensor_filter)) {

LOG_WARN("load lora tensors from %s failed", lora_spec.path.c_str());

if (lora_spec.required) {

return false;

}

continue;

}

if (lora->lora_tensors.empty()) {

if (lora_spec.required) {

LOG_ERROR("required lora has no tensors: %s", lora_spec.path.c_str());

return false;

}

continue;

}

lora->multiplier = lora_spec.multiplier;

if (!lora->apply(group.model_tensors, all_tensor_names, lora_version_, n_threads_, false))

return false;

lora->release_loaded_tensors();

}

for (TensorState* state : group.states) {

if (state != nullptr) {

state->applied_lora_epoch = current_lora_epoch_;

}

}

}

return true;

std::unordered_set<TensorState*> affected;

for (auto& state : tensor_states_) {

if (state->component != ModelComponent::LoRA && state->applied_lora_epoch != UINT64_MAX) {

affected.insert(state.get());

}

}

invalidate_sources(affected);

for (const auto& ignore_prefix : common_ignore_tensors_) {

if (starts_with(state.name, ignore_prefix)) {

return true;

}

}

return false;

if (state.tensor == nullptr || should_ignore(state) || is_optional_missing_tensor(state.name)) {

return true;

}

if (state.sources.empty()) {

LOG_ERROR("%s tensor '%s' not in model metadata", model_component_name(state.component), state.name.c_str());

return false;

}

TensorStorage tensor_storage = state.sources.front();

if (state.sources.size() > 1) {

const int dim = tensor_storage.n_dims - 1;

if (dim < 0 || dim >= GGML_MAX_DIMS) {

return false;

}

tensor_storage.ne[dim] = 0;

for (const auto& part : state.sources) {

if (part.n_dims != tensor_storage.n_dims || part.ne[dim] < 0 ||

part.ne[dim] > state.tensor->ne[dim] - tensor_storage.ne[dim]) {

LOG_ERROR("invalid tensor part '%s' for '%s'", part.name.c_str(), state.name.c_str());

return false;

}

for (int i = 0; i < GGML_MAX_DIMS; ++i) {

if (i != dim && part.ne[i] != tensor_storage.ne[i]) {

LOG_ERROR("incompatible tensor part '%s' for '%s'", part.name.c_str(), state.name.c_str());

return false;

}

}

tensor_storage.ne[dim] += part.ne[dim];

}

}

if (state.tensor->ne[0] != tensor_storage.ne[0] ||

state.tensor->ne[1] != tensor_storage.ne[1] ||

state.tensor->ne[2] != tensor_storage.ne[2] ||

state.tensor->ne[3] != tensor_storage.ne[3]) {

LOG_ERROR(

"%s tensor '%s' has wrong shape in model metadata: got [%d, %d, %d, %d], expected [%d, %d, %d, %d]",

model_component_name(state.component),

state.name.c_str(),

(int)tensor_storage.ne[0], (int)tensor_storage.ne[1], (int)tensor_storage.ne[2], (int)tensor_storage.ne[3],

(int)state.tensor->ne[0], (int)state.tensor->ne[1], (int)state.tensor->ne[2], (int)state.tensor->ne[3]);

return false;

}

return true;

std::vector<ParamsStorageBlock*>& created_storage_blocks) {

// A GPU that computes on mmapped params in place cannot address a CPU buffer, and nothing

// stages them for it, so they are mapped through a buffer of that GPU's device.

struct MmapGroup {

std::map<std::string, ggml_tensor*> candidates;

std::map<std::string, TensorState*> states;

};

std::map<ggml_backend_dev_t, MmapGroup> groups;

for (TensorState* state : states) {

if (state == nullptr || !can_mmap_storage(*state) || state->tensor == nullptr ||

state->tensor->data != nullptr || state->tensor->view_src != nullptr) {

continue;

}

ggml_backend_dev_t device = nullptr;

if (!sd_backend_is_cpu(state->compute_backend) && !sd_backend_is_cpu(state->params_backend)) {

device = ggml_backend_get_device(state->compute_backend);

}

MmapGroup& group = groups[device];

group.candidates[state->name] = state->tensor;

group.states[state->name] = state;

}

for (auto& [device, group] : groups) {

// Device buffers wrap read-only mappings only; params that LoRAs are merged into in place

// are loaded instead.

if (device != nullptr && writable_mmap_) {

continue;

}

auto mmap_store = model_loader_.mmap_tensors(group.candidates, {}, writable_mmap_, device);

if (mmap_store.empty()) {

continue;

}

auto block = std::make_unique<ParamsStorageBlock>();

block->mmap_tensor_stores = std::move(mmap_store);

ParamsStorageBlock* raw = block.get();

for (const auto& pair : group.states) {

TensorState* state = pair.second;

if (state != nullptr && state->tensor != nullptr && state->tensor->data != nullptr) {

block->states.push_back(state);

}

}

if (!block->states.empty()) {

params_storage_blocks_.push_back(std::move(block));

created_storage_blocks.push_back(raw);

}

}

return true;

if (state.sources.size() > 1 || state.source_file != 0 || !enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {

return false;

}

if (state.compute_backend == nullptr || state.params_backend == nullptr) {

return false;

}

return sd_backend_is_cpu(state.compute_backend) ||

sd_backend_is_cpu(state.params_backend) ||

backend_supports_host_buffer(state.compute_backend);

std::vector<ParamsStorageBlock*>& created_storage_blocks) {

std::map<std::pair<ggml_backend_buffer_type_t, int>, std::vector<TensorState*>> states_by_buffer_type;

for (TensorState* state : states) {

if (state == nullptr || state->tensor == nullptr) {

continue;

}

ggml_backend_buffer_type_t params_buft = params_buffer_type_for(*state);

if (params_buft == nullptr) {

return false;

}

states_by_buffer_type[{params_buft, static_cast<int>(state->residency_mode)}].push_back(state);

}

for (const auto& pair : states_by_buffer_type) {

ggml_backend_buffer_type_t params_buft = pair.first.first;

const std::vector<TensorState*>& states = pair.second;

size_t alignment = ggml_backend_buft_get_alignment(params_buft);

size_t max_size = ggml_backend_buft_get_max_size(params_buft);

if (!ggml_backend_buft_is_host(params_buft) &&

(max_size == 0 || max_size > MAX_RESIDENCY_BLOCK_BYTES)) {

max_size = MAX_RESIDENCY_BLOCK_BYTES;

}

auto alloc_chunk = [&](const std::vector<TensorState*>& chunk, size_t chunk_size) -> bool {

if (chunk.empty() || chunk_size == 0) {

return true;

}

ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(params_buft, chunk_size);

if (buffer == nullptr) {

LOG_ERROR("model manager alloc params backend buffer failed, size = %.2fMB",

chunk_size / (1024.0 * 1024.0));

return false;

}

ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);

std::vector<ggml_tensor*> initialized_tensors;

void* base = ggml_backend_buffer_get_base(buffer);

size_t offset = aligned_offset(base, 0, ggml_backend_buffer_get_alignment(buffer));

for (TensorState* state : chunk) {

ggml_tensor* tensor = state->tensor;

size_t tensor_size = GGML_PAD(ggml_backend_buffer_get_alloc_size(buffer, tensor),

ggml_backend_buffer_get_alignment(buffer));

enum ggml_status status = ggml_backend_tensor_alloc(buffer, tensor, static_cast<char*>(base) + offset);

if (status != GGML_STATUS_SUCCESS) {

LOG_ERROR("model manager failed to initialize params tensor '%s'", ggml_get_name(tensor));

for (ggml_tensor* initialized : initialized_tensors) {

initialized->buffer = nullptr;

initialized->data = nullptr;

initialized->extra = nullptr;

}

LOG_VERBOSE("model manager releasing params backend buffer (%6.2f MB, %zu tensors, %s)",

ggml_backend_buffer_get_size(buffer) / (1024.f * 1024.f),

initialized_tensors.size(),

ggml_backend_buffer_is_host(buffer) ? "RAM" : "VRAM");

ggml_backend_buffer_free(buffer);

return false;

}

initialized_tensors.push_back(tensor);

offset += tensor_size;

}

auto block = std::make_unique<ParamsStorageBlock>();

block->buffer = buffer;

block->states = chunk;

ParamsStorageBlock* raw = block.get();

params_storage_blocks_.push_back(std::move(block));

created_storage_blocks.push_back(raw);

return true;

};

std::vector<TensorState*> chunk;

size_t chunk_size = 0;

for (TensorState* state : states) {

ggml_tensor* tensor = state->tensor;

size_t tensor_size = GGML_PAD(ggml_backend_buft_get_alloc_size(params_buft, tensor), alignment);

// Some backends, e.g. Vulkan, report a preferred chunk size here rather than a

// hard per-tensor allocation limit. Oversized tensors are allocated alone.

if (!chunk.empty() && max_size > 0 && chunk_size + tensor_size > max_size) {

if (!alloc_chunk(chunk, chunk_size)) {

return false;

}

chunk.clear();

chunk_size = 0;

}

chunk.push_back(state);

chunk_size += tensor_size;

}

if (!alloc_chunk(chunk, chunk_size)) {

return false;

}

}

return true;

auto ctx = std::unique_ptr<ggml_context, decltype(&ggml_free)>(

ggml_init({state.sources.size() * ggml_tensor_overhead(), nullptr, true}), ggml_free);

if (!ctx) {

return false;

}

const size_t size = ggml_nbytes(state.tensor);

std::vector<uint8_t> buffer;

void* data = state.tensor->data;

if (!ggml_backend_buffer_is_host(state.tensor->buffer)) {

buffer.resize(size);

data = buffer.data();

}

std::map<std::string, ggml_tensor*> parts;

std::set<std::string> names;

size_t offset = 0;

for (const auto& source : state.sources) {

auto part = ggml_new_tensor(ctx.get(), state.tensor->type, source.n_dims, source.ne);

const size_t part_size = ggml_nbytes(part);

if (part_size > size - offset) {

return false;

}

part->data = static_cast<uint8_t*>(data) + offset;

parts[source.name] = part;

names.insert(source.name);

offset += part_size;

}

if (offset != size) {

return false;

}

std::set<std::string> loaded;

std::mutex mutex;

auto callback = [&](const TensorStorage& source, ggml_tensor** dst) {

*dst = nullptr;