danielhanchen · Oct 12, 2026 · Oct 11, 2026 · Oct 11, 2026 · Oct 11, 2026 · Oct 11, 2026
diff --git a/.github/workflows/unsloth-sd-prebuilt.yml b/.github/workflows/unsloth-sd-prebuilt.yml
@@ -273,16 +273,20 @@ jobs:
             arch: x86_64
             defines: "-DSD_METAL=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
             gate: "true"
+            cpu_floor: x64
           - label: Linux-Ubuntu-22.04-x86_64
             runner: ubuntu-22.04
             arch: x86_64
             defines: ""
             gate: "false"
+            cpu_floor: x64
           - label: Linux-Ubuntu-24.04-aarch64
             runner: ubuntu-24.04-arm
             arch: aarch64
-            defines: ""
+            # GCC 13 (the image default) rejects +sme, which the Armv9.2 variants need.
+            defines: "-DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14"
             gate: "false"
+            cpu_floor: armv8.0_1
           - label: Linux-Ubuntu-22.04-x86_64-vulkan
             runner: ubuntu-22.04
             arch: x86_64
@@ -328,20 +332,38 @@ jobs:
           echo "VULKAN_SDK=$sdk_root/x86_64" >> "$GITHUB_ENV"
           echo "$sdk_root/x86_64/bin" >> "$GITHUB_PATH"
 
+      # CPU bundles ship all ggml CPU variants, picked per host at startup; -DGGML_NATIVE=OFF alone
+      # pins AVX2 (SIGILL on older x86) or plain Armv8.0. Metal/Vulkan stay static.
       - name: Build sd-cli + sd-server
         working-directory: src
         run: |
           set -euo pipefail
+          CPU_VARIANTS=""
+          if [ -n "${{ matrix.cpu_floor }}" ]; then
+            CPU_VARIANTS="-DSD_BUILD_SHARED_GGML_LIB=ON -DBUILD_SHARED_LIBS=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
+            # macOS: @loader_path only, never an absolute rpath into the build tree.
+            if [ "$(uname -s)" = "Darwin" ]; then
+              CPU_VARIANTS="$CPU_VARIANTS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH=@loader_path"
+            fi
+          fi
           cmake -B build \
             -DCMAKE_BUILD_TYPE=Release \
             -DSD_BUILD_EXAMPLES=ON \
             -DSD_SERVER_BUILD_FRONTEND=OFF \
             -DSD_WEBP=OFF -DSD_WEBM=OFF \
             -DGGML_NATIVE=OFF \
+            $CPU_VARIANTS \
             ${{ matrix.defines }}
           if [ "$(uname -s)" = "Darwin" ]; then J="$(sysctl -n hw.logicalcpu)"; else J="$(nproc)"; fi
           cmake --build build --config Release -j "$J" --target sd-cli sd-server
 
+      - name: Pin the bundle's own ggml (CPU variants)
+        if: ${{ matrix.cpu_floor != '' && runner.os == 'Linux' }}
+        run: |
+          set -euo pipefail
+          sudo apt-get update -qq && sudo apt-get install -y -qq patchelf
+          bash tooling/scripts/unsloth/cpu_variants_gate.sh pin src/build/bin
+
       - name: macOS load gate
         if: ${{ matrix.gate == 'true' }}
         run: |
@@ -361,6 +383,10 @@ jobs:
           LICENSE_FILE: ${{ github.workspace }}/src/LICENSE
         run: python3 tooling/scripts/unsloth/package_bundle.py
 
+      - name: CPU variants gate
+        if: ${{ matrix.cpu_floor != '' }}
+        run: bash tooling/scripts/unsloth/cpu_variants_gate.sh check "dist/sd-${{ needs.resolve.outputs.tag }}-bin-${{ matrix.label }}.zip" "${{ matrix.cpu_floor }}"
+
       - name: Upload bundle
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
@@ -836,6 +862,7 @@ jobs:
         include:
           - label: win-cpu-x64
             vulkan: "OFF"
+            cpu_floor: x64
           - label: win-vulkan-x64
             vulkan: "ON"
     env:
@@ -895,14 +922,20 @@ jobs:
         shell: pwsh
         working-directory: src
         run: |
+          $cpuVariants = @()
+          if ('${{ matrix.cpu_floor }}' -ne '') {
+            $cpuVariants = @('-DSD_BUILD_SHARED_GGML_LIB=ON', '-DBUILD_SHARED_LIBS=ON', '-DGGML_BACKEND_DL=ON', '-DGGML_CPU_ALL_VARIANTS=ON')
+          }
           cmake -S . -B build -G Ninja `
             -DCMAKE_BUILD_TYPE=Release `
             -DCMAKE_CXX_FLAGS='/bigobj' `
             -DSD_BUILD_EXAMPLES=ON `
             -DSD_SERVER_BUILD_FRONTEND=OFF `
             -DSD_WEBP=OFF -DSD_WEBM=OFF `
             -DSD_VULKAN=${{ matrix.vulkan }} `
-            -DGGML_NATIVE=OFF
+            -DGGML_NATIVE=OFF `
+            @cpuVariants
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           cmake --build build --config Release -j 3 --target sd-cli sd-server
 
       # Before packaging, so every PE the bundle carries is signed. Windows
@@ -948,6 +981,44 @@ jobs:
           & tooling/.github/scripts/assert-windows-bundle-signed.ps1 -Path $zips
           exit $LASTEXITCODE
 
+      - name: CPU variants gate
+        if: ${{ matrix.cpu_floor != '' }}
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $zip = (Get-ChildItem dist -Filter *.zip | Select-Object -First 1).FullName
+          $work = Join-Path $env:RUNNER_TEMP 'cpu-variants'
+          $empty = Join-Path $env:RUNNER_TEMP 'cpu-variants-cwd'
+          Remove-Item -Recurse -Force $work, $empty -ErrorAction SilentlyContinue
+          New-Item -ItemType Directory -Force $work, $empty | Out-Null
+          Expand-Archive -Path $zip -DestinationPath $work
+          $cli = Get-ChildItem $work -Recurse -Filter sd-cli.exe | Select-Object -First 1
+          $dir = $cli.DirectoryName
+          $variants = @(Get-ChildItem $dir -Filter 'ggml-cpu-*.dll' | ForEach-Object { $_.BaseName -replace '^ggml-cpu-', '' })
+          Write-Host "variants: $($variants -join ' ')"
+          if ($variants -notcontains '${{ matrix.cpu_floor }}') {
+            Write-Host "::error::floor variant ${{ matrix.cpu_floor }} missing from the bundle"
+            exit 1
+          }
+          Push-Location $empty
+          $ErrorActionPreference = 'Continue'
+          $out = (& $cli.FullName -m (Join-Path $empty 'none.gguf') -p x -v 2>&1 | Out-String)
+          Pop-Location
+          $m = [regex]::Match($out, 'loaded CPU backend from (.+?\.dll)')
+          if (-not $m.Success) {
+            Write-Host $out
+            Write-Host '::error::sd-cli loaded no CPU backend variant'
+            exit 1
+          }
+          $loaded = $m.Groups[1].Value.Trim()
+          Write-Host "loaded: $loaded"
+          if ((Split-Path -Parent ([IO.Path]::GetFullPath($loaded))) -ne [IO.Path]::GetFullPath($dir)) {
+            Write-Host "::error::variant came from outside the bundle: $loaded"
+            exit 1
+          }
+          # sd-cli itself exits 1 at the missing model; that is the expected stop, not the verdict.
+          exit 0
+
       - name: Upload bundle
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:

diff --git a/scripts/unsloth/cpu_variants_gate.sh b/scripts/unsloth/cpu_variants_gate.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+# Copyright 2026-present the Unsloth AI Inc. team.
+#
+#   pin   <bin_dir>: Linux: DT_RPATH=$ORIGIN (beats LD_LIBRARY_PATH, unlike RUNPATH) so the bundle's ggml wins.
+#   check <bundle.zip> <floor_variant>: floor variant present, no build-tree paths, sd-cli loads a CPU variant from the bundle.
+
+set -euo pipefail
+
+mode="${1:?usage: cpu_variants_gate.sh pin <bin_dir> | check <bundle.zip> <floor_variant>}"
+
+case "$mode" in
+  pin)
+    bin_dir="${2:?bin_dir}"
+    [ "$(uname -s)" = "Linux" ] || { echo "pin: nothing to do on $(uname -s)"; exit 0; }
+    command -v patchelf > /dev/null || { echo "ERROR: patchelf not installed (the workflow installs it)" >&2; exit 1; }
+    n=0
+    for f in "$bin_dir"/sd-cli "$bin_dir"/sd-server "$bin_dir"/libggml*.so* "$bin_dir"/libstable-diffusion*.so*; do
+      [ -f "$f" ] && [ ! -L "$f" ] || continue
+      patchelf --force-rpath --set-rpath '$ORIGIN' "$f"
+      n=$((n + 1))
+    done
+    echo "pin: set DT_RPATH=\$ORIGIN on $n files"
+    ;;
+  check)
+    zip="${2:?bundle.zip}"
+    floor="${3:?floor variant, e.g. x64 or armv8.0_1}"
+    work=""
+    empty=""
+    cleanup() {
+      if [ -n "$work" ]; then rm -rf -- "$work"; fi
+      if [ -n "$empty" ]; then rm -rf -- "$empty"; fi
+    }
+    trap cleanup EXIT
+    work="$(mktemp -d "${RUNNER_TEMP:-/tmp}/cpu-variants-XXXXXX")"
+    unzip -q "$zip" -d "$work"
+    dir="$(dirname "$(find "$work" -type f \( -name sd-cli -o -name sd-cli.exe \) | head -n1)")"
+    [ -n "$dir" ] && [ -d "$dir" ] || { echo "ERROR: no sd-cli in $zip" >&2; exit 1; }
+    case "$(uname -s)" in
+      Darwin) ext=so ;;  # ggml names its MODULE libraries .so on macOS too
+      MINGW*|MSYS*|CYGWIN*) ext=dll ;;
+      *) ext=so ;;
+    esac
+    prefix="libggml-cpu-"
+    [ "$ext" = dll ] && prefix="ggml-cpu-"
+    variants=""
+    for f in "$dir"/"$prefix"*."$ext"; do
+      [ -f "$f" ] || continue
+      v="${f##*/}"
+      v="${v#"$prefix"}"
+      variants="$variants${v%."$ext"} "
+    done
+    echo "variants: $variants"
+    case " $variants " in
+      *" $floor "*) ;;
+      *) echo "ERROR: floor variant '$floor' missing; hosts below the next level would have no CPU backend" >&2; exit 1 ;;
+    esac
+    fail=0
+    if [ "$(uname -s)" = "Linux" ]; then
+      for f in "$dir"/sd-cli "$dir"/sd-server "$dir"/libggml*.so* "$dir"/libstable-diffusion*.so*; do
+        [ -f "$f" ] || continue
+        rp="$(readelf -d "$f" | sed -n 's/.*(\(RPATH\|RUNPATH\)).*\[\(.*\)\]/\1=\2/p')"
+        if [ "$rp" != 'RPATH=$ORIGIN' ]; then echo "ERROR: $(basename "$f") has '${rp:-no rpath}', want RPATH=\$ORIGIN" >&2; fail=1; fi
+      done
+    elif [ "$(uname -s)" = "Darwin" ]; then
+      for f in "$dir"/sd-cli "$dir"/sd-server "$dir"/*.dylib "$dir"/*.so; do
+        [ -f "$f" ] || continue
+        bad="$(otool -l "$f" | awk '/LC_RPATH/{getline; getline; print $2}' | grep -v '^@' || true)"
+        if [ -n "$bad" ]; then echo "ERROR: $(basename "$f") keeps a build-tree rpath: $bad" >&2; fail=1; fi
+        deps="$(otool -L "$f" | tail -n +2 | awk '{print $1}' | grep -E 'ggml|stable-diffusion' | grep -v '^@' || true)"
+        if [ -n "$deps" ]; then echo "ERROR: $(basename "$f") links ggml by absolute path: $deps" >&2; fail=1; fi
+      done
+    fi
+    cli="$dir/sd-cli"
+    [ -f "$cli.exe" ] && cli="$cli.exe"
+    # Nonexistent model: sd-cli loads backends then stops. Empty cwd defeats ggml's cwd search.
+    empty="$(mktemp -d "${RUNNER_TEMP:-/tmp}/cpu-variants-cwd-XXXXXX")"
+    out="$(cd "$empty" && env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH "$cli" -m "$empty/none.gguf" -p x -v 2>&1 || true)"
+    loaded="$(printf '%s\n' "$out" | sed -n 's/.*loaded CPU backend from \(.*\)$/\1/p' | head -n1 | tr -d '\r')"
+    printf '%s\n' "$out" | grep -A1 "System Info" | head -n 3 || true
+    if [ -z "$loaded" ]; then
+      echo "ERROR: sd-cli loaded no CPU backend variant" >&2
+      printf '%s\n' "$out" | tail -n 30 >&2
+      fail=1
+    else
+      echo "loaded: $loaded"
+      loaded_dir="$(cd "$(dirname "$loaded")" && pwd -P)"
+      want_dir="$(cd "$dir" && pwd -P)"
+      if [ "$loaded_dir" != "$want_dir" ]; then echo "ERROR: variant came from $loaded_dir, not the bundle" >&2; fail=1; fi
+    fi
+    exit "$fail"
+    ;;
+  *)
+    echo "unknown mode $mode" >&2
+    exit 2
+    ;;
+esac
diff --git a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch
@@ -0,0 +1,15 @@
+diff --git a/src/ggml-cpu/arch/x86/cpu-feats.cpp b/src/ggml-cpu/arch/x86/cpu-feats.cpp
+index d775a036..e3848609 100644
+--- a/src/ggml-cpu/arch/x86/cpu-feats.cpp
++++ b/src/ggml-cpu/arch/x86/cpu-feats.cpp
+@@ -300,6 +300,10 @@ static int ggml_backend_cpu_x86_score() {
+     if (!is.AVX512VL()) { return 0; }
+     if (!is.AVX512DQ()) { return 0; }
+     if (!is.AVX512BW()) { return 0; }
++    // Unsloth: the AVX512 variants are only offered on CPUs that also have AVX-VNNI (Zen 5,
++    // Sapphire Rapids and newer). On Zen 4, which has AVX512 but not AVX-VNNI, they ran
++    // stable-diffusion.cpp 15 to 50% slower than the AVX2 variant; the AVX2 one is picked there.
++    if (!is.AVX_VNNI()) { return 0; }
+     score += 1<<7;
+ #endif
+ #ifdef GGML_AVX512_VBMI
diff --git a/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch b/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch
@@ -0,0 +1,70 @@
+diff --git a/src/ggml-backend-reg.cpp b/src/ggml-backend-reg.cpp
+index 1c18b82..506593d 100644
+--- a/src/ggml-backend-reg.cpp
++++ b/src/ggml-backend-reg.cpp
+@@ -418,7 +418,8 @@ static fs::path get_executable_path() {
+     }
+     return base_path + "/";
+ #elif defined(__linux__) || defined(__FreeBSD__)
+-    std::string base_path = ".";
++    // Unsloth: no "." fallback; on failure return empty so the caller skips it (not the cwd).
++    std::string base_path;
+     std::vector<char> path(1024);
+     while (true) {
+         // get executable path
+@@ -428,7 +429,7 @@ static fs::path get_executable_path() {
+         ssize_t len = readlink("/proc/curproc/file", path.data(), path.size());
+ #    endif
+         if (len == -1) {
+-            break;
++            return {};
+         }
+         if (len < (ssize_t) path.size()) {
+             base_path = std::string(path.data(), len);
+@@ -444,10 +445,21 @@ static fs::path get_executable_path() {
+
+     return base_path + "/";
+ #elif defined(_WIN32)
++    // Unsloth: grow the buffer; a truncated result (len == size) would yield an ancestor directory.
+     std::vector<wchar_t> path(MAX_PATH);
+-    DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());
+-    if (len == 0) {
+-        return {};
++    DWORD len;
++    while (true) {
++        len = GetModuleFileNameW(NULL, path.data(), (DWORD) path.size());
++        if (len == 0) {
++            return {};
++        }
++        if (len < path.size()) {
++            break;
++        }
++        if (path.size() >= 32768) {
++            return {};
++        }
++        path.resize(path.size() * 2);
+     }
+     std::wstring base_path(path.data(), len);
+     // remove executable name
+@@ -488,14 +500,13 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
+ #ifdef GGML_BACKEND_DIR
+         search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));
+ #endif
+-        // default search paths: executable directory, current directory
+-        search_paths.push_back(get_executable_path());
+-        std::error_code cwd_ec;
+-        const fs::path cwd = fs::current_path(cwd_ec);
+-        if (cwd_ec) {
+-            GGML_LOG_DEBUG("%s: current_path() failure, error-message: %s\n", __func__, cwd_ec.message().c_str());
+-        } else {
+-            search_paths.push_back(cwd);
++        // Unsloth: default search path is the executable directory only. Every matching library is
++        // loaded (running its constructors) before it is scored, so searching the current directory
++        // would run any libggml-<name>-*.so placed wherever the user launches sd-cli / sd-server.
++        // Use GGML_BACKEND_DIR or an explicit path to load backends from elsewhere.
++        const fs::path executable_dir = get_executable_path();
++        if (!executable_dir.empty()) {
++            search_paths.push_back(executable_dir);
+         }
+     } else {
+         search_paths.push_back(fs::u8path(user_search_path));
diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp
@@ -211,6 +211,25 @@ ggml_tensor* ggml_ext_linear_matmul(ggml_context* ctx,
                                     ggml_tensor* x,
                                     ggml_tensor* w,
                                     bool force_prec_f32) {
+    if (sd_weight_may_be_cpu_repacked(w) && w->ne[2] == 1 && w->ne[3] == 1) {
+        // Repacked CPU matmuls (AMX, CPU_REPACK) get batch dims wrong and only take F32 activations
+        // (the type sd_cpu_repack_buffer_type_for probes with): feed contiguous 2D F32 rows.
+        if (x->type != GGML_TYPE_F32) {
+            x = ggml_cast(ctx, x, GGML_TYPE_F32);
+        }
+        if (!ggml_is_contiguous(x)) {
+            x = ggml_cont(ctx, x);
+        }
+        const int64_t ne1 = x->ne[1];
+        const int64_t ne2 = x->ne[2];
+        const int64_t ne3 = x->ne[3];
+        x                 = ggml_reshape_2d(ctx, x, x->ne[0], ne1 * ne2 * ne3);
+        x                 = ggml_mul_mat(ctx, w, x);
+        if (force_prec_f32) {
+            ggml_prec_set_acc(x, GGML_PREC_F32);
+        }
+        return ggml_reshape_4d(ctx, x, x->ne[0], ne1, ne2, ne3);
+    }
     if (x->ne[2] * x->ne[3] > 1024) {
         // workaround: avoid ggml cuda error
         int64_t ne2 = x->ne[2];