How to Run Underdog Saluki 27B: 2-Bit Qwen3.8-27B Tool-Calling Guide

In this tutorial, we work with Underdog Saluki 27B, a 7.9 GB build of Qwen3.8-27B that its makers say keeps tool calling intact at about two bits per weight and even edges out the full 54 GB model on their own 120-task benchmark. Saluki is built on ISTA-DASLab’s GSQ-RCO quantization of the same model, so we put both files side by side in stock llama.cpp. We start with the files themselves: a tensor-by-tensor diff of the two GGUF headers, then a byte-level comparison that shows whether Saluki is a remix of ISTA’s bits or a changed model. We then serve each file on one GPU, read its memory and speed, and score both on the same seeded 120 tasks from the Berkeley Function Calling Leaderboard, using BFCL’s own checker with strict and lenient parsing. Copy CodeCopiedUse a different Browser

import os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
 
RESULTS = {}          # step -> one-line result, for the summary table
CLAIMS = []           # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
 
 
def banner(title):
    print("\n" + "=" * 78)
    print(title)
    print("=" * 78)
 
 
def section(name):
    def wrap(fn):
        def run(*a, **kw):
            CURRENT[0] = name.split(".")[0]
            banner(name)
            t0 = time.time()
            try:
                out = fn(*a, **kw)
                RESULTS[name] = out if isinstance(out, str) else "ok"
                return out
            except Exception as e:
                RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
                print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
                traceback.print_exc(limit=3)
                return None
            finally:
                print(f"\n[{time.time() - t0:.1f} s]")
                json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
        return run
    return wrap
 
 
def claim(text, held):
    CLAIMS.append((CURRENT[0], text, held))
    return held
 
 
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
    os.makedirs(os.path.join(WORK, d), exist_ok=True)
 
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565"   # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
                         "61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
               "Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
                          "19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
    "Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
                   file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
                   sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
    "ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
                 file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
                 sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
           "berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
    "data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
    "data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
    "data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
    "data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
    "data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
    "data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
    "data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
    "data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
    "data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
    "eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
    "constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
    "constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
    "utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
    "model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
         ("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
         ("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
 
 
def sh(cmd, **kw):
    return subprocess.run(cmd, capture_output=True, text=True, **kw)
 
 
def sha256_file(path, chunk=1 

Everything the notebook touches is pinned before anything runs. The setup cell records the llama.cpp release, both model files and fourteen BFCL files by size and SHA-256, and it fixes the sampler for every request, because llama-server otherwise falls back to defaults we did not choose. It detects the platform, an NVIDIA GPU on Linux or an Apple-silicon Mac, and on Colab it ensures the CUDA 12 runtime libraries the prebuilt llama. cpp needs are importable.cpp needs are importable. Copy CodeCopiedUse a different Browser

@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
    global LLAMA_DIR, SERVER_BIN, SERVER_ENV
    name, size, sha = LLAMA_ASSET
    tar = os.path.join(WORK, "llama", name)
    fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
    with tarfile.open(tar) as t:
        t.extractall(os.path.join(WORK, "llama"), filter="data")
    SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
    LLAMA_DIR = os.path.dirname(SERVER_BIN)
    for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
        os.chmod(b, 0o755)
    SERVER_ENV = dict(os.environ)
    if CUDA_LIBS:
        SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
        missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
        assert not missing, f"unresolved libraries: {missing}"
    r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
    ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
    print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
 
    for rel, digest in BFCL_FILES.items():
        dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
        fetch(GORILLA + rel, dest, sha=digest)
    print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
 
    for tag, m in MODELS.items():
        path = os.path.join(WORK, "models", m["file"])
        had = os.path.exists(path) and os.path.getsize(path) == m["size"]
        url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
        dl = fetch(url, path, m["size"])          # size check now, hash below
        t0 = time.time()
        ok = sha256_file(path) == m["sha256"]
        how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
        print(f"{tag:7s}: {m['file']}  {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
              f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
        claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
        assert ok
    return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
 
 
download_step()

We download the CUDA build of llama.cpp b11565, both 2-bit files and the BFCL data, and check every byte against its pin. The Hugging Face downloads are resumable, and the hashes are the Hub’s own LFS digests at a fixed revision, so a silently replaced file would stop the notebook rather than change its results. BFCL’s task files, answer files, and checker code all come from one gorilla commit. The cell prints how long the downloads took, which is the slowest part of the run on Colab. Copy CodeCopiedUse a different Browser

GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
              16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
              21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
              30: ("BF16", 1, 2)}   # name, values per block, bytes per block (ggml-common.h)
 
 
def read_gguf(path):
    """GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
    f = open(path, "rb")
    u = lambda fmt: struct.unpack(" a["bytes"]; down += b["bytes"] 5s} {'weights':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'change MB':>10s}")
    for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]):
        if r["n"] {b}" for (a, b), n in moves.most_common()))
    print(f"   {down} get fewer bytes, {up} get more")
    claim("every tensor whose type changes gets fewer bits; none gets more", up == 0 and down == changed)
    untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6]
    print(f"   classes with no change at all: {', '.join(sorted(untouched))}")
    proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c
    claim("every attention and DeltaNet projection matrix keeps its type (only the small alpha/beta gates change)",
          all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"]))))
    keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t"))
    print(f"\nmetadata keys that differ: {keys}")
    print(f"Saluki's header still reads: general.name = {kv_s['general.name']!r}, quantized_by = {kv_s['general.quantized_by']!r}")
    print(f"sampling stored in both headers: temp {kv_s['general.sampling.temp']}, top_k {kv_s['general.sampling.top_k']}, "
          f"top_p {kv_s['general.sampling.top_p']:.2f} (these are Qwen's thinking-mode settings; step 5 shows what the server does with them)")
    saved = (MODELS["ISTA"]["s