{
  "title": "256K model x engine benchmark",
  "source": "https://omarshabab.com/local-llm-256k-leaderboard/",
  "machine": "Mac Studio M3 Ultra, 512 GB unified memory, 80 GPU cores, macOS 26.3",
  "measured_between": "2026-08-10 to 2026-08-27",
  "benchmark": "needle in a haystack, 8 needles at evenly spaced depths, greedy decoding",
  "notes": {
    "depth": "Tokens actually served. Rows differ because some prompts pass through a chat template and some do not.",
    "prefill_decode": "Tokens per second. Decode is a least-squares slope over per-token timestamps, not wall-clock division.",
    "memory": "Two bases, stated per row and NOT interchangeable. \"alloc\" is the MLX device allocator peak for that process; \"sys\" is the whole machine peak footprint.",
    "mhz": "Median GPU clock over the run, from per-second macmon samples.",
    "ranges": "Where a value is a range it spans repeated runs of an identical configuration. The numeric field carries the LOW end.",
    "recall": "Scored on the answer segment, against the built prompt own needle record. A generation that opens a reasoning block, never closes it, and hits the token cap is treated as unmeasurable rather than as a failure."
  },
  "measured": [
    {
      "model": "Qwen3.5-2B",
      "engine": "mlx-lm bf16",
      "depth": "234,158",
      "prefill": 2026.6,
      "prefillLabel": "2026.6",
      "decode": 65.31,
      "decodeLabel": "65.31",
      "recall": "8/8",
      "memory": "56.5 GiB sys",
      "mhz": "1376",
      "winner": false,
      "flagged": true
    },
    {
      "model": "Nemotron-3.5-Lightning-30B-A3B",
      "engine": "mlx-lm 8-bit",
      "depth": "262,223",
      "prefill": 986,
      "prefillLabel": "986",
      "decode": 60.45,
      "decodeLabel": "60.45",
      "recall": "8/8",
      "memory": "38.7 GB alloc",
      "mhz": "1379",
      "winner": true,
      "flagged": false
    },
    {
      "model": "Qwopus3.5-4B-Coder",
      "engine": "mlx-lm 4-bit",
      "depth": "234,158",
      "prefill": 484,
      "prefillLabel": "484 to 719",
      "decode": 37.97,
      "decodeLabel": "37.97",
      "recall": "8/8",
      "memory": "27.9 GB alloc",
      "mhz": "1372 to 1377",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3-Next-80B-A3B",
      "engine": "mlx-lm 4-bit",
      "depth": "262,257",
      "prefill": 397,
      "prefillLabel": "397",
      "decode": 26.79,
      "decodeLabel": "26.79",
      "recall": "8/8",
      "memory": "71.1 GB alloc",
      "mhz": "1153",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.8-27B-abliterated-oQ4e",
      "engine": "oMLX 0.6.2 + DFlash2",
      "depth": "234,158",
      "prefill": 102,
      "prefillLabel": "102.0",
      "decode": 18.61,
      "decodeLabel": "18.61",
      "recall": "8/8",
      "memory": "133.4 GiB sys",
      "mhz": "732",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.6-35B-A3B",
      "engine": "mlx-lm 8-bit",
      "depth": "234,158",
      "prefill": 519,
      "prefillLabel": "519",
      "decode": 17.7,
      "decodeLabel": "17.7 to 26.4",
      "recall": "8/8",
      "memory": "114.6 GiB sys",
      "mhz": "1193",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen2.5-7B-Instruct-1M",
      "engine": "mlx-lm 8-bit",
      "depth": "262,268",
      "prefill": 191.6,
      "prefillLabel": "191.6",
      "decode": 11.67,
      "decodeLabel": "11.67",
      "recall": "8/8",
      "memory": "24.53 GB alloc",
      "mhz": "729",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3-4B-Instruct-2507",
      "engine": "mlx-lm 4-bit",
      "depth": "262,257",
      "prefill": 175,
      "prefillLabel": "175",
      "decode": 10.15,
      "decodeLabel": "10.15",
      "recall": "8/8",
      "memory": "43.7 GB alloc",
      "mhz": "890",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.6-27B",
      "engine": "mlx-lm 8-bit",
      "depth": "234,158",
      "prefill": 114,
      "prefillLabel": "114.0",
      "decode": 7.89,
      "decodeLabel": "7.89",
      "recall": "8/8",
      "memory": "132.3 GiB sys",
      "mhz": "708",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.8-27B-Abliterated",
      "engine": "mlx-lm 8-bit",
      "depth": "262,302",
      "prefill": 0,
      "prefillLabel": "not reportable",
      "decode": 6.96,
      "decodeLabel": "6.96",
      "recall": "8/8",
      "memory": "75.5 GB alloc",
      "mhz": "695",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.6-35B-A3B",
      "engine": "oMLX 0.5.5",
      "depth": "234,158",
      "prefill": 696.7,
      "prefillLabel": "696.7",
      "decode": 6.86,
      "decodeLabel": "6.86",
      "recall": "8/8",
      "memory": "87.6 GiB sys",
      "mhz": "not recorded",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.6-27B",
      "engine": "llama.cpp Q8_0",
      "depth": "234,158",
      "prefill": 95.6,
      "prefillLabel": "95.6",
      "decode": 5.1,
      "decodeLabel": "5.10",
      "recall": "8/8",
      "memory": "85.2 GiB sys",
      "mhz": "655",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.8-27B-abliterated-oQ4e",
      "engine": "oMLX 0.6.2 + dual-ANE + MTP k=3",
      "depth": "234,158",
      "prefill": 147,
      "prefillLabel": "147.0",
      "decode": 3.98,
      "decodeLabel": "3.98",
      "recall": "8/8",
      "memory": "115.6 GiB sys",
      "mhz": "902",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.8-27B-abliterated-oQ4e",
      "engine": "oMLX 0.6.2 baseline",
      "depth": "234,158",
      "prefill": 159,
      "prefillLabel": "159.0",
      "decode": 3.91,
      "decodeLabel": "3.91",
      "recall": "8/8",
      "memory": "115.7 GiB sys",
      "mhz": "957",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.6-27B",
      "engine": "oMLX 0.5.5",
      "depth": "234,158",
      "prefill": 128.7,
      "prefillLabel": "128.7",
      "decode": 3.66,
      "decodeLabel": "3.66",
      "recall": "8/8",
      "memory": "101.9 GiB sys",
      "mhz": "not recorded",
      "winner": false,
      "flagged": false
    },
    {
      "model": "Qwen3.6-27B",
      "engine": "surge (my C + Metal engine)",
      "depth": "234,158",
      "prefill": 2.99,
      "prefillLabel": "2.99 compute",
      "decode": 0.537,
      "decodeLabel": "0.537",
      "recall": "8/8",
      "memory": "47.0 GiB alloc",
      "mhz": "684",
      "winner": false,
      "flagged": true
    }
  ],
  "failures": [
    {
      "model": "GLM-5.3-Flash (320B-A18B)",
      "engine": "glm53_flash_mlx (patched mlx-vlm), mixed 4/8-bit",
      "depth": "234,143",
      "prefill": "42.3, limiter-shaped",
      "decode": "14.83",
      "recall": "8/8 retrieval and association, never terminates",
      "memory": "270.11 GB alloc / 366.7 GiB sys",
      "mhz": "504, 11 pct clamped"
    },
    {
      "model": "Qwen3.8-27B, BASE weights",
      "engine": "mlx-lm 8-bit",
      "depth": "262,302",
      "prefill": "105.3",
      "decode": "7.12",
      "recall": "FAIL, never terminates",
      "memory": "76 GB alloc",
      "mhz": "504"
    },
    {
      "model": "Qwen2.5-7B-Instruct-1M",
      "engine": "mlx-lm 8-bit",
      "depth": "955,201",
      "prefill": "50.2",
      "decode": "2.89",
      "recall": "FAIL 0/8, degenerate",
      "memory": "151.8 GiB sys",
      "mhz": "see caveats"
    },
    {
      "model": "Falcon-H1-7B",
      "engine": "mlx-lm 4-bit",
      "depth": "about 262k",
      "prefill": "not reportable",
      "decode": "not reportable",
      "recall": "FAIL 0/8, degenerate",
      "memory": "not reportable",
      "mhz": "not reportable"
    }
  ],
  "published_claims_at_their_real_depth": [
    {
      "claim": "oMLX dual-ANE recipe, as published",
      "engine": "oMLX 0.6.1 dual-ANE",
      "depth": "2,048",
      "prefill": "431.8",
      "decode": "81.1"
    },
    {
      "claim": "DFlash2, as published",
      "engine": "various engines, M5 Max",
      "depth": "short context",
      "prefill": "not published",
      "decode": "2.7x to 3.4x speedup"
    }
  ],
  "not_measured": [
    {
      "model": "DeepSeek-V4-Flash (304B-A13B)",
      "reason": "Blocked on stock mlx-lm by a runtime gap in fp4-packed MXFP4 expert dequantization. An engine limitation, not a model result."
    },
    {
      "model": "Ling-3.0-tiny",
      "reason": "Ineligible. Downloaded as a candidate, then found to have a trained context of 131,072, so it never belonged in a 262k comparison."
    }
  ]
}