{"table": "Recipe", "generated": "2026-11-11T14:40:39Z", "license": "CC BY-SA 4.0", "docs": "https://cmp.wiki/wiki/CMP_Wiki:For_agents", "fields": ["pagetitle", "page", "model", "quant", "engine", "engine_version", "cards", "card_count", "topology", "context", "decode_tps", "prefill_tps", "vram_per_card", "power_cap", "driver", "unlock_build", "flags", "status", "tested_by", "tested_on", "source", "files", "model_family", "engine_family", "decode_single", "decode_agg", "agg_users", "prefill_prompt", "context_tokens", "p2p", "pcie_link", "pcie_width", "power_w", "spec_decode", "bench_note"], "keys": "Row keys match the names in \"fields\" (underscores, for example decode_tps). The older spelling with a space (\"decode tps\") is kept in every row with the same value, so existing readers keep working.", "count": 47, "rows": [{"pagetitle": "DeepSeek V4 Flash 0731 on 3x 8 GB (PP3 DSpark)", "page": "DeepSeek V4 Flash 0731 on 3x 8 GB (PP3 DSpark)", "model": "DeepSeek-V4-Flash-0731", "quant": "Native MXFP4 experts + FP8 attention (official deepseek-ai checkpoint, about 155.4 GiB); KV cache fp8", "engine": "vLLM: haosdent/vllm branch dsv4-flash-a100 with a vendored copy of allover326's 7 patches (image dsv4-a100:devel)", "engine_version": "f8ea5bb-era base with the allover326 patch set as of mid-August 2026", "cards": "8 GB", "card_count": "3", "topology": "PP3 (VLLM_PP_LAYER_PARTITION=15,15,13) with DSpark, 5 draft tokens; PCIe Gen2 x4, no P2P", "context": "691,968 (largest stable context at --gpu-memory-utilization 0.965)", "decode_tps": "93", "prefill_tps": "", "vram_per_card": "64 GB (65,536 MiB), --gpu-memory-utilization 0.965", "power_cap": "100 W", "driver": "610.43.02 (nvidia-driver-610-open)", "unlock_build": "cmpunlocker 360acd7 (2026-08-06), 8 GB profile", "flags": "--pipeline-parallel-size 3 --kv-cache-dtype fp8 --block-size 256 --max-model-len 691968 --max-num-batched-tokens 2048 --gpu-memory-utilization 0.965 --max-num-seqs 8 --tokenizer-mode deepseek_v4 --no-enable-flashinfer-autotune --enable-auto-tool-choice --tool-call-parser deepseek_v4 --reasoning-parser deepseek_v4, DSpark speculative config with num_speculative_tokens 5, top_p 0.95 generation override; env VLLM_PP_LAYER_PARTITION=15,15,13, DSV4_LOGITS_ROW_CHUNK=64, VLLM_SPARSE_DENSE_QUERY_BLOCK=0, VLLM_WORKER_MULTIPROC_METHOD=spawn; container --shm-size=16g", "status": "Reported", "tested_by": "tim-odonnell", "tested_on": "2026-08-15", "source": "https://github.com/tim-odonnell/cmp170hx-deepseek-v4-flash", "files": "vllm-dsv4/phase5-launch-dspark-production.sh, HOWTO-RUN-DEEPSEEK-V4-FLASH.md, PERFORMANCE-SUMMARY.md (the repo has no license file)", "model_family": "DeepSeek V4 Flash", "engine_family": "vLLM", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "691968", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "100", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "", "tested_on__precision": "1", "engine version": "f8ea5bb-era base with the allover326 patch set as of mid-August 2026", "card count": "3", "decode tps": "93", "prefill tps": "", "vram per card": "64 GB (65,536 MiB), --gpu-memory-utilization 0.965", "power cap": "100 W", "unlock build": "cmpunlocker 360acd7 (2026-08-06), 8 GB profile", "tested by": "tim-odonnell", "tested on": "2026-08-15", "model family": "DeepSeek V4 Flash", "engine family": "vLLM", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "691968", "pcie link": "Gen2", "pcie width": "x4", "power w": "100", "spec decode": "DSpark, 5 draft tokens", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4 Flash 0731 on 4x 8 GB (allover326 PP4 DSpark)", "page": "DeepSeek V4 Flash 0731 on 4x 8 GB (allover326 PP4 DSpark)", "model": "DeepSeek-V4-Flash-0731", "quant": "Native MXFP4 experts + FP8 attention (official deepseek-ai checkpoint, about 155.4 GiB); KV cache fp8", "engine": "vLLM: haosdent/vllm branch dsv4-flash-a100 plus allover326's 7 patches (DSpark under PP, sm_80 prefill top-k fallback, logits row chunking)", "engine_version": "base c3046d1 (recommended since 2026-08-13; first released on f8ea5bb); repo commit 3dd2d88", "cards": "8 GB", "card_count": "4", "topology": "PP4 (VLLM_PP_LAYER_PARTITION=12,12,12,7) with DSpark, 5 draft tokens; PCIe Gen2 x4, no P2P", "context": "1,047,736 verified one-shot; 1,002,852 over a 405-turn chat with DSV4_LOGITS_ROW_CHUNK=64", "decode_tps": "98.1", "prefill_tps": "5300", "vram_per_card": "64 GB, --gpu-memory-utilization 0.85", "power_cap": "180 W", "driver": "", "unlock_build": "cmpunlocker VRAM unlock (65,536 MiB per card)", "flags": "--pipeline-parallel-size 4 --kv-cache-dtype fp8 --block-size 256 --max-num-batched-tokens 2048 --max-num-seqs 8 --trust-remote-code --no-enable-flashinfer-autotune --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4, DSpark speculative config with num_speculative_tokens 5; env VLLM_WORKER_MULTIPROC_METHOD=spawn, DSV4_LOGITS_ROW_CHUNK=64, VLLM_PP_LAYER_PARTITION=12,12,12,7; container --shm-size=16g; never --enforce-eager", "status": "Proven", "tested_by": "deejay9983", "tested_on": "2026-08-05", "source": "https://github.com/allover326/deepseek-v4-cmp170hx", "files": "launch/run-pp-dspark.sh, patches 0001 to 0006 plus 0005a, docker/Dockerfile.devel and docker/Dockerfile.fullbuild (Apache-2.0)", "model_family": "DeepSeek V4 Flash", "engine_family": "vLLM", "decode_single": "98.1", "decode_agg": "712.8", "agg_users": "64", "prefill_prompt": "25k to 77k tokens", "context_tokens": "1047736", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "180", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "repo bench, temperature 0, prefix caching off, ignore_eos on", "tested_on__precision": "1", "engine version": "base c3046d1 (recommended since 2026-08-13; first released on f8ea5bb); repo commit 3dd2d88", "card count": "4", "decode tps": "98.1", "prefill tps": "5300", "vram per card": "64 GB, --gpu-memory-utilization 0.85", "power cap": "180 W", "unlock build": "cmpunlocker VRAM unlock (65,536 MiB per card)", "tested by": "deejay9983", "tested on": "2026-08-05", "model family": "DeepSeek V4 Flash", "engine family": "vLLM", "decode single": "98.1", "decode agg": "712.8", "agg users": "64", "prefill prompt": "25k to 77k tokens", "context tokens": "1047736", "pcie link": "Gen2", "pcie width": "x4", "power w": "180", "spec decode": "DSpark, 5 draft tokens", "bench note": "repo bench, temperature 0, prefix caching off, ignore_eos on", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4 Flash pruned on 4x 10 GB", "page": "DeepSeek V4 Flash pruned on 4x 10 GB", "model": "DeepSeek V4 Flash with 25% of the experts pruned (base checkpoint not stated)", "quant": "", "engine": "vLLM (version and flags not stated)", "engine_version": "", "cards": "10 GB", "card_count": "4", "topology": "not tensor parallel (the split is not stated)", "context": "", "decode_tps": "48.2", "prefill_tps": "", "vram_per_card": "about 37 GB usable per card (40 GB profile), 148 GB in total", "power_cap": "250 W limit (read on 2026-08-11, during his llama.cpp runs)", "driver": "", "unlock_build": "", "flags": "", "status": "Reported", "tested_by": "cuddylac997", "tested_on": "2026-08-12", "source": "https://discord.com/channels/1518634379604136178/1518695328008638516/1537123473309306910", "files": "", "model_family": "DeepSeek V4 Flash", "engine_family": "vLLM", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "", "p2p": "", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "", "spec_decode": "none", "bench_note": "one post (number of users not stated)", "tested_on__precision": "1", "engine version": "", "card count": "4", "decode tps": "48.2", "prefill tps": "", "vram per card": "about 37 GB usable per card (40 GB profile), 148 GB in total", "power cap": "250 W limit (read on 2026-08-11, during his llama.cpp runs)", "unlock build": "", "tested by": "cuddylac997", "tested on": "2026-08-12", "model family": "DeepSeek V4 Flash", "engine family": "vLLM", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "", "pcie link": "Gen2", "pcie width": "x4", "power w": "", "spec decode": "none", "bench note": "one post (number of users not stated)", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash heretic on 4x 8 GB", "page": "DeepSeek V4.1 Flash heretic on 4x 8 GB", "model": "DeepSeek-V4.1-Flash-1017 heretic (pqhaz abliteration)", "quant": "Hybrid: routed experts EXL3 2.9 bpw (LibertAIDAI), everything else official FP8/BF16 with the 38 heretic attn.wo_b tensors", "engine": "Morrowmake/vllm-cmp170hx (vLLM fork, container image)", "engine_version": "recipe 1.0.0 (commit d4d1f525), engine a0c57b52", "cards": "8 GB", "card_count": "4", "topology": "TP4", "context": "262,144", "decode_tps": "158", "prefill_tps": "2453", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.92 (miner co-resident)", "power_cap": "", "driver": "", "unlock_build": "", "flags": "EXTRA_ARGS=\"--gpu-memory-utilization 0.92\", VLLM_ENGRAM_PLACEMENT=ssd, CUDA_DEVICE_ORDER=PCI_BUS_ID", "status": "Reported", "tested_by": "darkmaniac7", "tested_on": "2026-10-29", "source": "https://github.com/Morrowmake/dsv41-flash-cmp170hx-recipe", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "158", "decode_agg": "323", "agg_users": "8", "prefill_prompt": "17k tokens", "context_tokens": "262144", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "", "bench_note": "lab's own bench script, code column, miner co-resident on the cards", "tested_on__precision": "1", "engine version": "recipe 1.0.0 (commit d4d1f525), engine a0c57b52", "card count": "4", "decode tps": "158", "prefill tps": "2453", "vram per card": "64 GiB, --gpu-memory-utilization 0.92 (miner co-resident)", "power cap": "", "unlock build": "", "tested by": "darkmaniac7", "tested on": "2026-10-29", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "158", "decode agg": "323", "agg users": "8", "prefill prompt": "17k tokens", "context tokens": "262144", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "", "bench note": "lab's own bench script, code column, miner co-resident on the cards", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 2x 8 GB (Project Maya)", "page": "DeepSeek V4.1 Flash on 2x 8 GB (Project Maya)", "model": "DeepSeek-V4.1-Flash-1017 (748B in the Maya README)", "quant": "GSQ-RCO 2-bit GGUF (this row); Maya GGUF (Maya-S 340 GB or Maya-XS 277 GB; Engram tables stay on SSD)", "engine": "Project Maya (Strata and llama.cpp lineage)", "engine_version": "SeriousRising's patched fork, branch rw-comb5 (this row)", "cards": "8 GB", "card_count": "2", "topology": "engine-managed expert tiering across VRAM, host RAM and NVMe", "context": "", "decode_tps": "47", "prefill_tps": "1190", "vram_per_card": "64 GB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "", "status": "Reported", "tested_by": "seriousrising", "tested_on": "2026-10-28", "source": "https://github.com/SeriousRising/ds41-flash-cmp170hx", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "Project Maya", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "4K tokens", "context_tokens": "", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "", "bench_note": "one chat, temperature 1, thinking high; about 52 at temperature 0, thinking off (author's README)", "tested_on__precision": "1", "engine version": "SeriousRising's patched fork, branch rw-comb5 (this row)", "card count": "2", "decode tps": "47", "prefill tps": "1190", "vram per card": "64 GB", "power cap": "", "unlock build": "", "tested by": "seriousrising", "tested on": "2026-10-28", "model family": "DeepSeek V4.1 Flash", "engine family": "Project Maya", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "4K tokens", "context tokens": "", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "", "bench note": "one chat, temperature 1, thinking high; about 52 at temperature 0, thinking off (author's README)", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 4x 8 GB (alanoo fidelity build)", "page": "DeepSeek V4.1 Flash on 4x 8 GB (alanoo fidelity build)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Hybrid: all 40 expert layers at 3 bits (LibertAI 2.9 bpw pack with layers 18-22 grafted from the 3.5 bpw pack), official FP8 dense tensors", "engine": "GNmac-lab recipe image 1.2.0 (vLLM with sm_80 patches) plus three patches", "engine_version": "GNmac-lab at 2ad5741ab1; image ds41-cmp170hx:1.2.0-fidelity", "cards": "8 GB", "card_count": "4", "topology": "TP4, P2P on", "context": "262,144", "decode_tps": "105", "prefill_tps": "2514", "vram_per_card": "64 GiB (64,911 MiB usable), GPU_MEMORY_UTILIZATION 0.98", "power_cap": "200 W", "driver": "NVIDIA open 610.43.02 from a cmpunlocker fork with a P2P patch", "unlock_build": "cmpunlocker fork with P2P (without Morrowmake's TRAP31 patch)", "flags": "LAYOUT=tp4, MAX_MODEL_LEN=262144, MAX_NUM_SEQS=8, GPU_MEMORY_UTILIZATION=0.98, P2P=1, SPEC_TOKENS=0 (DSpark off), ENGRAM_STORAGE=ssd from a tmpfs", "status": "Reported", "tested_by": "alanoo", "tested_on": "2026-10-27", "source": "https://github.com/alanoo81/deepseek-v41-flash-4x-cmp170hx", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "105", "decode_agg": "482.4", "agg_users": "8", "prefill_prompt": "24k to 38k tokens", "context_tokens": "262144", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "200", "spec_decode": "none", "bench_note": "MiaAI-Lab, temperature 0, thinking off, 400-token cap, code column, DSpark off", "tested_on__precision": "1", "engine version": "GNmac-lab at 2ad5741ab1; image ds41-cmp170hx:1.2.0-fidelity", "card count": "4", "decode tps": "105", "prefill tps": "2514", "vram per card": "64 GiB (64,911 MiB usable), GPU_MEMORY_UTILIZATION 0.98", "power cap": "200 W", "unlock build": "cmpunlocker fork with P2P (without Morrowmake's TRAP31 patch)", "tested by": "alanoo", "tested on": "2026-10-27", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "105", "decode agg": "482.4", "agg users": "8", "prefill prompt": "24k to 38k tokens", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "200", "spec decode": "none", "bench note": "MiaAI-Lab, temperature 0, thinking off, 400-token cap, code column, DSpark off", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 4x 8 GB (CurtisAccelerate TC-MoE)", "page": "DeepSeek V4.1 Flash on 4x 8 GB (CurtisAccelerate TC-MoE)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Hybrid: LibertAI EXL3 2.9 bpw routed experts (layers 0-39), official FP8/FP4 for attention, shared experts, head, draft and Engram; KV fp8_ds_mla", "engine": "Morrowmake/vllm-cmp170hx (bind-mounted sm80 package) plus the author's tensor-core MoE kernels", "engine_version": "vllm-cmp170hx 18305948ac; TC-MoE fb9786ec", "cards": "8 GB", "card_count": "4", "topology": "TP4, peer access off (NCCL_P2P_DISABLE=1), host all-reduce", "context": "262,144 (weights address 1,048,576)", "decode_tps": "144.2", "prefill_tps": "1523", "vram_per_card": "about 62.5 GB used of 64 GB, --gpu-memory-utilization 0.97", "power_cap": "140 W", "driver": "", "unlock_build": "", "flags": "--quantization exl3 --tensor-parallel-size 4 --max-num-seqs 8 --max-num-batched-tokens 2048 --gpu-memory-utilization 0.97 --kv-cache-dtype fp8_ds_mla --engram-config cpu_offload --speculative-config dspark 5", "status": "Reported", "tested_by": "CurtisAccelerate", "tested_on": "2026-10-22", "source": "https://github.com/CurtisAccelerate/DeepSeekv41-Flash-1017-EXL3-CMP170HX", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "144.2", "decode_agg": "271.6", "agg_users": "8", "prefill_prompt": "2,048 tokens", "context_tokens": "262144", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "140", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "author's bench, thinking off, temp 0, code column; 8-user total from 10-24 comparison", "tested_on__precision": "1", "engine version": "vllm-cmp170hx 18305948ac; TC-MoE fb9786ec", "card count": "4", "decode tps": "144.2", "prefill tps": "1523", "vram per card": "about 62.5 GB used of 64 GB, --gpu-memory-utilization 0.97", "power cap": "140 W", "unlock build": "", "tested by": "CurtisAccelerate", "tested on": "2026-10-22", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "144.2", "decode agg": "271.6", "agg users": "8", "prefill prompt": "2,048 tokens", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "140", "spec decode": "DSpark, 5 draft tokens", "bench note": "author's bench, thinking off, temp 0, code column; 8-user total from 10-24 comparison", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 4x 8 GB (GNmac with copy fix and faster prefill)", "page": "DeepSeek V4.1 Flash on 4x 8 GB (GNmac with copy fix and faster prefill)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Hybrid as recipe 1.2.0: routed experts EXL3 from LibertAI's 2.9 bpw pack (mostly 3 bpw, 2 bpw on layers 18-22); attention, shared experts, indexer and DSpark drafter official FP8; KV fp8_ds_mla", "engine": "GNmac-lab's vLLM sm_80 build (recipe 1.2.0) plus the copy fix (pull request 2) and two prefill changes (hiacc MoE prefill, collision-free owner sum)", "engine_version": "recipe 1.2.0 (tag v1.2.0, 2ad5741) + pull request 2 (cad5cc6) + prefill patch set 2026-11-09", "cards": "8 GB", "card_count": "4", "topology": "TP4 (LAYOUT=tp4); the prefill changes are TP4 only", "context": "262,144", "decode_tps": "283", "prefill_tps": "2732", "vram_per_card": "64 GiB, GPU_MEMORY_UTILIZATION 0.95", "power_cap": "180 W (nvidia-smi -pl 180)", "driver": "", "unlock_build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable driver, as recipe 1.2.0", "flags": "Recipe 1.2.0 settings (LAYOUT=tp4, MAX_MODEL_LEN 262144, MAX_NUM_SEQS 8, GPU_MEMORY_UTILIZATION 0.95, P2P 1, DSV41_GATE_TAU 0.10, DSV41_DYNK accept:3,5) plus DSV41_MOE_TRY_SO=/opt/dsv41-pf/_dsv41_pf_hiacc.abi3.so, DSV41_MOE_TRY_NS=_dsv41_pf_hiacc and DSV41_CE_CF=tokR", "status": "Reported", "tested_by": "darkmaniac7", "tested_on": "2026-11-09", "source": "https://github.com/GNmac-lab/deepseek-v41-flash-1017-cmp170hx", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "283", "decode_agg": "559", "agg_users": "8", "prefill_prompt": "24k, 34k and 38k tokens, cold, median of 9", "context_tokens": "262144", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "180", "spec_decode": "DSpark, 5 drafts, confidence gate tau 0.10, DYNK accept:3,5 (as recipe 1.2.0)", "bench_note": "the recipe's bench/mia.py (MiaAI-Lab protocol), code column, mean of 2 runs; prefill by the recipe's bench/prefill.py on a fresh instance; same session as a 1.2.0 control", "tested_on__precision": "1", "engine version": "recipe 1.2.0 (tag v1.2.0, 2ad5741) + pull request 2 (cad5cc6) + prefill patch set 2026-11-09", "card count": "4", "decode tps": "283", "prefill tps": "2732", "vram per card": "64 GiB, GPU_MEMORY_UTILIZATION 0.95", "power cap": "180 W (nvidia-smi -pl 180)", "unlock build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable driver, as recipe 1.2.0", "tested by": "darkmaniac7", "tested on": "2026-11-09", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "283", "decode agg": "559", "agg users": "8", "prefill prompt": "24k, 34k and 38k tokens, cold, median of 9", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "180", "spec decode": "DSpark, 5 drafts, confidence gate tau 0.10, DYNK accept:3,5 (as recipe 1.2.0)", "bench note": "the recipe's bench/mia.py (MiaAI-Lab protocol), code column, mean of 2 runs; prefill by the recipe's bench/prefill.py on a fresh instance; same session as a 1.2.0 control", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 4x 8 GB (GNmac)", "page": "DeepSeek V4.1 Flash on 4x 8 GB (GNmac)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Hybrid: routed experts EXL3 (mostly 3 bpw, 2 bpw on layers 18-22) from LibertAI's 2.9 bpw pack; attention, shared experts, indexer and DSpark drafter official FP8; KV fp8_ds_mla", "engine": "vLLM with a pinned sm_80 patch stack (built locally from source)", "engine_version": "recipe 1.2.0", "cards": "8 GB", "card_count": "4", "topology": "TP4 (default) or PP4 for x1/x4 links", "context": "262,144 (up to 1,048,576)", "decode_tps": "258", "prefill_tps": "2416", "vram_per_card": "64 GiB, GPU_MEMORY_UTILIZATION 0.95", "power_cap": "180 W (nvidia-smi -pl 180)", "driver": "CUDA 13 capable; measured on 610.43.02", "unlock_build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable driver for TP4", "flags": "LAYOUT=tp4, MAX_MODEL_LEN 262144, MAX_NUM_SEQS 8, GPU_MEMORY_UTILIZATION 0.95, P2P 1, DSV41_GATE_TAU 0.10, DSV41_DYNK accept:3,5", "status": "Proven", "tested_by": "greencandle9753", "tested_on": "2026-10-26", "source": "https://github.com/GNmac-lab/deepseek-v41-flash-1017-cmp170hx", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "258", "decode_agg": "544", "agg_users": "8", "prefill_prompt": "", "context_tokens": "262144", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "180", "spec_decode": "DSpark, 5 drafts, confidence gate tau 0.10, DYNK accept:3,5", "bench_note": "MiaAI-Lab protocol, code column, release 1.2.0 TP4", "tested_on__precision": "1", "engine version": "recipe 1.2.0", "card count": "4", "decode tps": "258", "prefill tps": "2416", "vram per card": "64 GiB, GPU_MEMORY_UTILIZATION 0.95", "power cap": "180 W (nvidia-smi -pl 180)", "unlock build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable driver for TP4", "tested by": "greencandle9753", "tested on": "2026-10-26", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "258", "decode agg": "544", "agg users": "8", "prefill prompt": "", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "180", "spec decode": "DSpark, 5 drafts, confidence gate tau 0.10, DYNK accept:3,5", "bench note": "MiaAI-Lab protocol, code column, release 1.2.0 TP4", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 4x 8 GB (Morrowmake)", "page": "DeepSeek V4.1 Flash on 4x 8 GB (Morrowmake)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Hybrid: routed experts EXL3 2.9 bpw (LibertAIDAI), everything else official FP8/BF16; KV BF16", "engine": "Morrowmake/vllm-cmp170hx (vLLM fork, container image)", "engine_version": "recipe 1.0.1 (2026-11-01), image sha256:32ae8b0e91a6; 1.0.0 was engine a0c57b5237; 1.0.2 (2026-11-02, image sha256:73e79648b5e2) is a startup fix only, numbers unchanged", "cards": "8 GB", "card_count": "4", "topology": "TP4; 1.0.1 splits every routed expert across the four cards (1.0.0: 96 whole experts per card)", "context": "262,144", "decode_tps": "297", "prefill_tps": "3776", "vram_per_card": "64 GiB exposed, --gpu-memory-utilization 0.95 (fixed by the release since 1.0.1)", "power_cap": "180 W", "driver": "NVIDIA 580 or newer; P2P needs the Morrowmake/cmpunlocker build", "unlock_build": "cmpunlocker (64 GiB, 74 SMs); Morrowmake/cmpunlocker for P2P", "flags": "P2P=auto, VLLM_ENGRAM_PLACEMENT=auto (ram or ssd), MAX_SEQS 8, MAX_BATCHED 2048, DSpark drafter up to 7 tokens", "status": "Reported", "tested_by": "voidgiraffe", "tested_on": "2026-11-01", "source": "https://github.com/Morrowmake/dsv41-flash-cmp170hx-recipe", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "297", "decode_agg": "795", "agg_users": "8", "prefill_prompt": "2,048 tokens", "context_tokens": "262144", "p2p": "optional", "pcie_link": "", "pcie_width": "x16", "power_w": "180", "spec_decode": "DSpark, up to 7 draft tokens for one request (5 for more)", "bench_note": "MiaAI-Lab, 400 tokens, temperature 0, thinking off, median of 5 runs, code column", "tested_on__precision": "1", "engine version": "recipe 1.0.1 (2026-11-01), image sha256:32ae8b0e91a6; 1.0.0 was engine a0c57b5237; 1.0.2 (2026-11-02, image sha256:73e79648b5e2) is a startup fix only, numbers unchanged", "card count": "4", "decode tps": "297", "prefill tps": "3776", "vram per card": "64 GiB exposed, --gpu-memory-utilization 0.95 (fixed by the release since 1.0.1)", "power cap": "180 W", "unlock build": "cmpunlocker (64 GiB, 74 SMs); Morrowmake/cmpunlocker for P2P", "tested by": "voidgiraffe", "tested on": "2026-11-01", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "297", "decode agg": "795", "agg users": "8", "prefill prompt": "2,048 tokens", "context tokens": "262144", "pcie link": "", "pcie width": "x16", "power w": "180", "spec decode": "DSpark, up to 7 draft tokens for one request (5 for more)", "bench note": "MiaAI-Lab, 400 tokens, temperature 0, thinking off, median of 5 runs, code column", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 4x 8 GB plus RTX PRO 6000 (ExLlamaV3 TP5)", "page": "DeepSeek V4.1 Flash on 4x 8 GB plus RTX PRO 6000 (ExLlamaV3 TP5)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "EXL3 3.5 bpw (LibertAIDAI; routed experts 4 bpw on layers 0-9 and 30-39, 3 bpw on 10-29; attention and shared experts 6 bpw) plus the official Engram shards", "engine": "ExLlamaV3 fork (anoane/exllamav3-anemone) + TabbyAPI fork (anoane/tabbyAPI)", "engine_version": "tag recipe-09 (309ef6d1); TabbyAPI deepseek-v41 (ce250ff)", "cards": "8 GB", "card_count": "4", "topology": "TP5 across four CMP 170HX and one RTX PRO 6000 Blackwell (native backend, reduces through host memory); MTP draft on the PRO", "context": "1,048,576 FP16", "decode_tps": "99.9", "prefill_tps": "2116", "vram_per_card": "gpu_split 58 GB per CMP, 70 GB on the PRO", "power_cap": "150 W per CMP (PRO 400 W)", "driver": "NVIDIA open 610.57.04 patched with Morrowmake/cmpunlocker 1f2c498 (profile 8gb)", "unlock_build": "Morrowmake/cmpunlocker 1f2c498; modprobe RmForceEnableGen2=1, RMPcieLinkSpeed=0x1, ForceP2P=0x11", "flags": "'\"`UNIQ--nowiki-0000007A-QINU`\"'", "status": "Reported", "tested_by": "anoane", "tested_on": "2026-10-20", "source": "https://github.com/anoane/09-deepseek-v41-flash-1017-exl3-tp5-1m", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "ExLlamaV3", "decode_single": "99.9", "decode_agg": "", "agg_users": "", "prefill_prompt": "26,296 tokens", "context_tokens": "1048576", "p2p": "off", "pcie_link": "", "pcie_width": "x16", "power_w": "150", "spec_decode": "MTP (draft head on the RTX PRO 6000, dynamic draft)", "bench_note": "nine C-coding requests (139 to 9,426 tokens), mean decode, max_batch_size 1", "tested_on__precision": "1", "engine version": "tag recipe-09 (309ef6d1); TabbyAPI deepseek-v41 (ce250ff)", "card count": "4", "decode tps": "99.9", "prefill tps": "2116", "vram per card": "gpu_split 58 GB per CMP, 70 GB on the PRO", "power cap": "150 W per CMP (PRO 400 W)", "unlock build": "Morrowmake/cmpunlocker 1f2c498; modprobe RmForceEnableGen2=1, RMPcieLinkSpeed=0x1, ForceP2P=0x11", "tested by": "anoane", "tested on": "2026-10-20", "model family": "DeepSeek V4.1 Flash", "engine family": "ExLlamaV3", "decode single": "99.9", "decode agg": "", "agg users": "", "prefill prompt": "26,296 tokens", "context tokens": "1048576", "pcie link": "", "pcie width": "x16", "power w": "150", "spec decode": "MTP (draft head on the RTX PRO 6000, dynamic draft)", "bench note": "nine C-coding requests (139 to 9,426 tokens), mean decode, max_batch_size 1", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 5x 8 GB (PP5 full residency)", "page": "DeepSeek V4.1 Flash on 5x 8 GB (PP5 full residency)", "model": "DeepSeek-V4.1-Flash", "quant": "Official checkpoint (510 GB: MXFP4 routed experts, FP8 dense layers, 203 GB FP8 Engram table); KV cache fp8_ds_mla", "engine": "vLLM: 344303947's fork of wtdcode/vllm-backport (shadow source layer, staged Marlin repack, exact-size Engram pinning)", "engine_version": "v0.13.1.dev6, tag v41-pp5-1m-pinmem-20260916", "cards": "8 GB", "card_count": "5", "topology": "PP5 with an even 8,8,8,8,8 layer split; the repo's cards are on PCIe Gen2 x16 with no NVLink", "context": "1,048,576 (KV pool 1,632,827 tokens at utilisation 0.97; an 896K-token prompt passed)", "decode_tps": "", "prefill_tps": "4600", "vram_per_card": "64 GB, full residency (no expert offload), --gpu-memory-utilization 0.97", "power_cap": "", "driver": "", "unlock_build": "", "flags": "start_vllm_ds4.1.sh --maxlen 1048576 --gpu-util 0.97 --offload-ranks 0,0,0,0,0; vLLM --pipeline-parallel-size 5 --tensor-parallel-size 1 --kv-cache-dtype fp8_ds_mla --max-num-batched-tokens 2048 --max-num-seqs 8 --enable-prefix-caching --disable-custom-all-reduce, Engram CPU offload; env VLLM_PP_LAYER_PARTITION=8,8,8,8,8, VLLM_ENGRAM_EXACT_PIN (default on), VLLM_SPARSE_INDEXER_MAX_LOGITS_MB=128, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True, NCCL_ALGO=Ring, NCCL_PROTO=Simple", "status": "Reported", "tested_by": "344303947", "tested_on": "2026-09-22", "source": "https://github.com/344303947/dsv41-flash-pp5-170hx", "files": "dsv41-pp5/ (Chinese docs: README.md, ARCHITECTURE.md, CHANGELOG.md, all-changes.patch, start_vllm_ds4.1.sh; Apache-2.0)", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "60K tokens", "context_tokens": "1048576", "p2p": "", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "", "spec_decode": "", "bench_note": "", "tested_on__precision": "1", "engine version": "v0.13.1.dev6, tag v41-pp5-1m-pinmem-20260916", "card count": "5", "decode tps": "", "prefill tps": "4600", "vram per card": "64 GB, full residency (no expert offload), --gpu-memory-utilization 0.97", "power cap": "", "unlock build": "", "tested by": "344303947", "tested on": "2026-09-22", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "60K tokens", "context tokens": "1048576", "pcie link": "Gen2", "pcie width": "x16", "power w": "", "spec decode": "", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 6x 8 GB (Deepinfer-SM80 TP2 x PP3)", "page": "DeepSeek V4.1 Flash on 6x 8 GB (Deepinfer-SM80 TP2 x PP3)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Official checkpoint (FP8 dense, FP4 experts); software FP8 on sm_80", "engine": "vLLM PR #56201 head (79a7108d) plus the two Deepinfer-SM80 patches and 30 drop-in modules", "engine_version": "Deepinfer-SM80 2a828fc lineage; vLLM v0.28.1rc0 base", "cards": "8 GB", "card_count": "6", "topology": "TP2 x PP3, partition 14,14,12", "context": "1,048,576", "decode_tps": "160.8", "prefill_tps": "4037", "vram_per_card": "64 GiB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "VLLM_PP_LAYER_PARTITION=14,14,12, --speculative-config dspark num_speculative_tokens 5, --max-model-len 1048576", "status": "Reported", "tested_by": "HalfVulpes", "tested_on": "2026-10-28", "source": "https://github.com/HalfVulpes/Deepinfer-SM80", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "160.8", "decode_agg": "", "agg_users": "", "prefill_prompt": "16K tokens", "context_tokens": "1048576", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "one stream on code, 1M max context", "tested_on__precision": "1", "engine version": "Deepinfer-SM80 2a828fc lineage; vLLM v0.28.1rc0 base", "card count": "6", "decode tps": "160.8", "prefill tps": "4037", "vram per card": "64 GiB", "power cap": "", "unlock build": "", "tested by": "HalfVulpes", "tested on": "2026-10-28", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "160.8", "decode agg": "", "agg users": "", "prefill prompt": "16K tokens", "context tokens": "1048576", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "DSpark, 5 draft tokens", "bench note": "one stream on code, 1M max context", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 6x 8 GB (TP6)", "page": "DeepSeek V4.1 Flash on 6x 8 GB (TP6)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Official checkpoint: dense MXFP8 through an SM80 Marlin W8A16 path, routed experts MXFP4 through W4A16; KV FP8", "engine": "Custom SM80 vLLM (patched from public commit 79a7108d9)", "engine_version": "reports 0.0.0; patches/public-base-to-runtime.patch (109 files)", "cards": "8 GB", "card_count": "6", "topology": "TP6 (virtual-head padding), PP1", "context": "1,048,576", "decode_tps": "78.55", "prefill_tps": "", "vram_per_card": "64 GB, --gpu-memory-utilization 0.95", "power_cap": "", "driver": "", "unlock_build": "", "flags": "--tensor-parallel-size 6 --kv-cache-dtype fp8 --block-size 128 --max-num-seqs 16 --max-num-batched-tokens 2048 --engram-config storage disk --speculative-config dspark 5; --hf-overrides virtual heads 96 / o_groups 12", "status": "Reported", "tested_by": "wgkbfcp", "tested_on": "2026-10-19", "source": "https://github.com/wgkbfcp/deepseek-v4.1-flash-1017-cmp170hx-recipe", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "78.55", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "1048576", "p2p": "", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "single request, code, end to end incl. reasoning, effort max, temperature 1, Engram warm", "tested_on__precision": "1", "engine version": "reports 0.0.0; patches/public-base-to-runtime.patch (109 files)", "card count": "6", "decode tps": "78.55", "prefill tps": "", "vram per card": "64 GB, --gpu-memory-utilization 0.95", "power cap": "", "unlock build": "", "tested by": "wgkbfcp", "tested on": "2026-10-19", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "78.55", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "1048576", "pcie link": "Gen2", "pcie width": "x16", "power w": "", "spec decode": "DSpark, 5 draft tokens", "bench note": "single request, code, end to end incl. reasoning, effort max, temperature 1, Engram warm", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 8x 8 GB (Deepinfer-SM80 TP4 x PP2)", "page": "DeepSeek V4.1 Flash on 8x 8 GB (Deepinfer-SM80 TP4 x PP2)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Official checkpoint (FP8 dense, FP4 experts); software FP8 on sm_80", "engine": "vLLM PR #56201 head (79a7108d) plus the two Deepinfer-SM80 patches and 30 drop-in modules", "engine_version": "Deepinfer-SM80, vLLM v0.28.1rc0 base", "cards": "8 GB", "card_count": "8", "topology": "TP4 x PP2, partition 20,20", "context": "1,048,576", "decode_tps": "153.35", "prefill_tps": "3463", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.95", "power_cap": "", "driver": "", "unlock_build": "", "flags": "VLLM_PP_LAYER_PARTITION=20,20, --gpu-memory-utilization 0.95, max_num_seqs 12, DSpark K=5", "status": "Reported", "tested_by": "HalfVulpes", "tested_on": "2026-10-28", "source": "https://github.com/HalfVulpes/Deepinfer-SM80", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "153.35", "decode_agg": "620.22", "agg_users": "12", "prefill_prompt": "", "context_tokens": "1048576", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "concurrency 1 on code; concurrency 12 aggregate (workload not stated)", "tested_on__precision": "1", "engine version": "Deepinfer-SM80, vLLM v0.28.1rc0 base", "card count": "8", "decode tps": "153.35", "prefill tps": "3463", "vram per card": "64 GiB, --gpu-memory-utilization 0.95", "power cap": "", "unlock build": "", "tested by": "HalfVulpes", "tested on": "2026-10-28", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "153.35", "decode agg": "620.22", "agg users": "12", "prefill prompt": "", "context tokens": "1048576", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "DSpark, 5 draft tokens", "bench note": "concurrency 1 on code; concurrency 12 aggregate (workload not stated)", "tested on__precision": "1"}, {"pagetitle": "DeepSeek V4.1 Flash on 8x 8 GB (Zanooda PP8)", "page": "DeepSeek V4.1 Flash on 8x 8 GB (Zanooda PP8)", "model": "DeepSeek-V4.1-Flash-1017", "quant": "Official checkpoint as downloaded: MXFP4 routed experts as Marlin W4A16, MXFP8 dense as Marlin W8A16; KV fp8", "engine": "vLLM PR #56201 head (79a7108d) plus a 20-patch sm_80 series; image zanooda/vllm-sm80-ds41f:v41-sm80", "engine_version": "recipe 31318fd; patches 0001-0020", "cards": "8 GB", "card_count": "8", "topology": "PP8, partition 5,5,5,5,5,5,5,5", "context": "1,048,576", "decode_tps": "117", "prefill_tps": "6066", "vram_per_card": "64 GB; weights 35.9 GB per rank (ranks 1-6), 37.6 GB rank 0, 46.3 GB rank 7", "power_cap": "180 W (one card unstable at 250 W)", "driver": "", "unlock_build": "cmpunlocker with the BAR1 P2P patches (64 GiB BAR1 per card)", "flags": "--pipeline-parallel-size 8 --max-model-len 1048576 --gpu-memory-utilization 0.95 --block-size 128 --kv-cache-dtype fp8 --max-num-batched-tokens 4096 --max-num-seqs 32; DSpark 5; VLLM_DSV41_SPREAD_DECODE=2; --engram-config storage cpu", "status": "Reported", "tested_by": "Zanooda", "tested_on": "2026-10-19", "source": "https://github.com/Zanooda/deepseek-v4.1-flash-cmp170hx", "files": "", "model_family": "DeepSeek V4.1 Flash", "engine_family": "vLLM", "decode_single": "117", "decode_agg": "560", "agg_users": "8", "prefill_prompt": "104,881 tokens", "context_tokens": "1048576", "p2p": "optional", "pcie_link": "Gen2", "pcie_width": "mixed", "power_w": "", "spec_decode": "DSpark, 5 draft tokens", "bench_note": "single stream at 128k context, Engram in RAM; 8 streams warm with patch 0019", "tested_on__precision": "1", "engine version": "recipe 31318fd; patches 0001-0020", "card count": "8", "decode tps": "117", "prefill tps": "6066", "vram per card": "64 GB; weights 35.9 GB per rank (ranks 1-6), 37.6 GB rank 0, 46.3 GB rank 7", "power cap": "180 W (one card unstable at 250 W)", "unlock build": "cmpunlocker with the BAR1 P2P patches (64 GiB BAR1 per card)", "tested by": "Zanooda", "tested on": "2026-10-19", "model family": "DeepSeek V4.1 Flash", "engine family": "vLLM", "decode single": "117", "decode agg": "560", "agg users": "8", "prefill prompt": "104,881 tokens", "context tokens": "1048576", "pcie link": "Gen2", "pcie width": "mixed", "power w": "", "spec decode": "DSpark, 5 draft tokens", "bench note": "single stream at 128k context, Engram in RAM; 8 streams warm with patch 0019", "tested on__precision": "1"}, {"pagetitle": "GLM 5.2 on 3x 10 GB (llama.cpp CPU offload)", "page": "GLM 5.2 on 3x 10 GB (llama.cpp CPU offload)", "model": "GLM-5.2 (unsloth/GLM-5.2-GGUF, about 460 GB of weights)", "quant": "Q4 GGUF (the exact Q4 file is not named); UD-Q2_K_XL also tested", "engine": "llama-server with partial CPU offload (llama.cpp or ik_llama.cpp: the build is not named)", "engine_version": "", "cards": "10 GB", "card_count": "3", "topology": "partial offload: part of the model on the three cards, the rest in host RAM", "context": "", "decode_tps": "6.39", "prefill_tps": "48.22", "vram_per_card": "40 GB profile (10 GB card unlocked to 40 GB)", "power_cap": "", "driver": "", "unlock_build": "10 GB to 40 GB unlock", "flags": "", "status": "Reported", "tested_by": "eksitus", "tested_on": "2026-07-24", "source": "https://discord.com/channels/1518634379604136178/1518695289278566450/1530298177910210690", "files": "", "model_family": "GLM 5.2", "engine_family": "llama.cpp", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "2048 tokens", "context_tokens": "", "p2p": "", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "", "spec_decode": "", "bench_note": "pp2048 and tg512 (benchmark tool and number of users not stated)", "tested_on__precision": "1", "engine version": "", "card count": "3", "decode tps": "6.39", "prefill tps": "48.22", "vram per card": "40 GB profile (10 GB card unlocked to 40 GB)", "power cap": "", "unlock build": "10 GB to 40 GB unlock", "tested by": "eksitus", "tested on": "2026-07-24", "model family": "GLM 5.2", "engine family": "llama.cpp", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "2048 tokens", "context tokens": "", "pcie link": "Gen2", "pcie width": "x4", "power w": "", "spec decode": "", "bench note": "pp2048 and tg512 (benchmark tool and number of users not stated)", "tested on__precision": "1"}, {"pagetitle": "GLM 5.2 on 8x 8 GB", "page": "GLM 5.2 on 8x 8 GB", "model": "GLM-5.2 (the full model, not Flash)", "quant": "4-bit, posted as int4 and q4 (the checkpoint is named in Notes)", "engine": "vLLM with a sparse-attention (DSA) backend for sm_80", "engine_version": "", "cards": "8 GB", "card_count": "8", "topology": "PP8 (pipeline parallel); tensor parallel ruled out at Gen1 x4", "context": "", "decode_tps": "30", "prefill_tps": "2600", "vram_per_card": "64 GB (8 GB card unlocked)", "power_cap": "", "driver": "610.43.02", "unlock_build": "", "flags": "--pipeline-parallel-size 8 --gpu-memory-utilization 0.90", "status": "Reported", "tested_by": "deejay9983", "tested_on": "2026-07-27", "source": "https://discord.com/channels/1518634379604136178/1518695289278566450/1531333858337755336", "files": "", "model_family": "GLM 5.2", "engine_family": "vLLM", "decode_single": "30", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "", "p2p": "", "pcie_link": "Gen1", "pcie_width": "x4", "power_w": "", "spec_decode": "none", "bench_note": "rounded figures from the tester's chat posts; one session on rented cards", "tested_on__precision": "1", "engine version": "", "card count": "8", "decode tps": "30", "prefill tps": "2600", "vram per card": "64 GB (8 GB card unlocked)", "power cap": "", "unlock build": "", "tested by": "deejay9983", "tested on": "2026-07-27", "model family": "GLM 5.2", "engine family": "vLLM", "decode single": "30", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "", "pcie link": "Gen1", "pcie width": "x4", "power w": "", "spec decode": "none", "bench note": "rounded figures from the tester's chat posts; one session on rented cards", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 Flash on 2x 8 GB (ExLlamaV3 layer split)", "page": "GLM 5.3 Flash on 2x 8 GB (ExLlamaV3 layer split)", "model": "GLM-5.3-Flash (320B MoE, 18B active)", "quant": "EXL3 3.05 bpw: turboderp/GLM-5.3-Flash-exl3", "engine": "ExLlamaV3 1.5.4 (built for SM80) with a small OpenAI-compatible server", "engine_version": "Flun/glm53-flash-cmp170hx-exl3 at 5933112", "cards": "8 GB", "card_count": "2", "topology": "layer split (no tensor parallel), no P2P", "context": "392,960 (profile q8_384k)", "decode_tps": "60", "prefill_tps": "", "vram_per_card": "about 58 GiB of 64 GiB (116.6 GiB resident across both)", "power_cap": "", "driver": "", "unlock_build": "", "flags": "scripts/serve.sh --profile q8_384k (-cs 393216 -cq 8 -gs 62,63 --q8-staging --q8-head-gpu0 -chunk_size 2048)", "status": "Proven", "tested_by": "68x0", "tested_on": "2026-10-20", "source": "https://github.com/Flun/glm53-flash-cmp170hx-exl3", "files": "", "model_family": "GLM 5.3 Flash", "engine_family": "ExLlamaV3", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "392960", "p2p": "off", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "", "bench_note": "", "tested_on__precision": "1", "engine version": "Flun/glm53-flash-cmp170hx-exl3 at 5933112", "card count": "2", "decode tps": "60", "prefill tps": "", "vram per card": "about 58 GiB of 64 GiB (116.6 GiB resident across both)", "power cap": "", "unlock build": "", "tested by": "68x0", "tested on": "2026-10-20", "model family": "GLM 5.3 Flash", "engine family": "ExLlamaV3", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "392960", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 Flash on 2x 8 GB (ExLlamaV3 TP2)", "page": "GLM 5.3 Flash on 2x 8 GB (ExLlamaV3 TP2)", "model": "GLM-5.3-Flash (320B MoE, 18B active)", "quant": "EXL3 3.05 bpw: turboderp/GLM-5.3-Flash-exl3; DFlash2 drafter converted to EXL3 6 bpw", "engine": "ExLlamaV3 (patched, pinned)", "engine_version": "upstream dev 0f685aa9fd plus 9 patches; repo db01c256", "cards": "8 GB", "card_count": "2", "topology": "TP2 over NCCL (GPU P2P), DFlash2 K7 plus hybrid drafting", "context": "392,960 per request; 655,360-token Q8 KV pool shared by two requests", "decode_tps": "107.67", "prefill_tps": "1533", "vram_per_card": "about 62.9 / 62.3 GiB idle, peaks about 64.4 / 63.0 GiB with vision", "power_cap": "180 W (author's cards); about 170 W with a modest memory OC in his Discord numbers", "driver": "610.57.04", "unlock_build": "cmpunlocker (64 GiB); P2P-capable driver for the NCCL backend", "flags": "scripts/run_tp.sh: -cq 8 -cs 655360 --context-cap 392960 -ndt 7 -chunk_size 2048 -ambs 2 -gs 62,63 -tp --experimental-dflash2-tp -tpb nccl --q8-staging --q8-head-gpu0 --concurrent-max-sequences 2; EXL3_MLA_KIDX_RING=32", "status": "Reported", "tested_by": "nestaa51", "tested_on": "2026-10-13", "source": "https://github.com/alexvnesta/glm53-flash-cmp170hx-tp", "files": "", "model_family": "GLM 5.3 Flash", "engine_family": "ExLlamaV3", "decode_single": "107.67", "decode_agg": "67.25", "agg_users": "2", "prefill_prompt": "32K tokens", "context_tokens": "392960", "p2p": "optional", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "", "spec_decode": "DFlash2, 7 draft tokens, with hybrid drafting (15 tokens)", "bench_note": "Author's qualified single-request bench, short code task; 2-stream workload not stated", "tested_on__precision": "1", "engine version": "upstream dev 0f685aa9fd plus 9 patches; repo db01c256", "card count": "2", "decode tps": "107.67", "prefill tps": "1533", "vram per card": "about 62.9 / 62.3 GiB idle, peaks about 64.4 / 63.0 GiB with vision", "power cap": "180 W (author's cards); about 170 W with a modest memory OC in his Discord numbers", "unlock build": "cmpunlocker (64 GiB); P2P-capable driver for the NCCL backend", "tested by": "nestaa51", "tested on": "2026-10-13", "model family": "GLM 5.3 Flash", "engine family": "ExLlamaV3", "decode single": "107.67", "decode agg": "67.25", "agg users": "2", "prefill prompt": "32K tokens", "context tokens": "392960", "pcie link": "Gen2", "pcie width": "x16", "power w": "", "spec decode": "DFlash2, 7 draft tokens, with hybrid drafting (15 tokens)", "bench note": "Author's qualified single-request bench, short code task; 2-stream workload not stated", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 Flash on 2x 8 GB (Project Maya)", "page": "GLM 5.3 Flash on 2x 8 GB (Project Maya)", "model": "GLM-5.3-Flash (321B in the Maya README)", "quant": "Maya-M GGUF (116 GB), every expert in VRAM; Maya-S (96.5 GB) and Maya-S24 (94.7 GB) also reported", "engine": "Project Maya (Strata and llama.cpp lineage, EXL3 loading, DFlash2 drafter)", "engine_version": "v1.0.6 (this row)", "cards": "8 GB", "card_count": "2", "topology": "engine-managed split with expert tiering across VRAM, RAM and NVMe", "context": "", "decode_tps": "61.3", "prefill_tps": "514", "vram_per_card": "64 GB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "./maya.sh --bench (three greedy 256-token answers for decode, an 8K-token prompt for prefill)", "status": "Reported", "tested_by": "ZackO2o", "tested_on": "", "source": "https://github.com/mw00/project-maya#how-fast-is-it", "files": "", "model_family": "GLM 5.3 Flash", "engine_family": "Project Maya", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "8K tokens", "context_tokens": "", "p2p": "", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "", "spec_decode": "", "bench_note": "maya.sh --bench, three greedy 256-token answers (ZackO2o, v1.0.6, README user report)", "tested_on__precision": "", "engine version": "v1.0.6 (this row)", "card count": "2", "decode tps": "61.3", "prefill tps": "514", "vram per card": "64 GB", "power cap": "", "unlock build": "", "tested by": "ZackO2o", "tested on": "", "model family": "GLM 5.3 Flash", "engine family": "Project Maya", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "8K tokens", "context tokens": "", "pcie link": "Gen2", "pcie width": "x4", "power w": "", "spec decode": "", "bench note": "maya.sh --bench, three greedy 256-token answers (ZackO2o, v1.0.6, README user report)", "tested on__precision": ""}, {"pagetitle": "GLM 5.3 Flash on 3x 8 GB", "page": "GLM 5.3 Flash on 3x 8 GB", "model": "GLM-5.3-Flash (320B MoE, 18B active)", "quant": "W4A16, about 4.3 bpw, 163 GiB on disk", "engine": "Morrowmake/vllm-cmp170hx (via the glm53-flash-cmp170hx-recipe)", "engine_version": "not stated", "cards": "8 GB", "card_count": "3", "topology": "PP3", "context": "about 1,000,000 (8-bit KV cache)", "decode_tps": "", "prefill_tps": "", "vram_per_card": "64 GiB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "Morrowmake's recipe with pipeline parallel 3; KV cache quantized to 8 bits", "status": "Reported", "tested_by": "shirogeek", "tested_on": "2026-10-19", "source": "https://discord.com/channels/1518634379604136178/1518695289278566450/1561743826945843304", "files": "", "model_family": "GLM 5.3 Flash", "engine_family": "vLLM", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "", "bench_note": "", "tested_on__precision": "1", "engine version": "not stated", "card count": "3", "decode tps": "", "prefill tps": "", "vram per card": "64 GiB", "power cap": "", "unlock build": "", "tested by": "shirogeek", "tested on": "2026-10-19", "model family": "GLM 5.3 Flash", "engine family": "vLLM", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 Flash on 4x 8 GB", "page": "GLM 5.3 Flash on 4x 8 GB", "model": "GLM-5.3-Flash (320B MoE, 18B active)", "quant": "W4A16 (INT4 weights, group size 128, FP16 activations): canada-quant/GLM-5.3-Flash-W4A16-MTP", "engine": "Morrowmake/vllm-cmp170hx (vLLM fork, container image)", "engine_version": "recipe 1.8.0, engine 819146023f", "cards": "8 GB", "card_count": "4", "topology": "TP4 (default) or PP4", "context": "262,144", "decode_tps": "442.1", "prefill_tps": "3161", "vram_per_card": "64 GiB exposed, --gpu-memory-utilization 0.95", "power_cap": "180 W", "driver": "NVIDIA 580 or newer; P2P needs the Morrowmake/cmpunlocker build", "unlock_build": "cmpunlocker (64 GiB, 74 SMs); Morrowmake/cmpunlocker for P2P", "flags": "LAYOUT=tp4 or LAYOUT=pp4, P2P=auto, --max-num-seqs 8, --gpu-memory-utilization 0.95, --max-num-batched-tokens 3463 (TP4) or 2312 (PP4)", "status": "Proven", "tested_by": "voidgiraffe", "tested_on": "2026-10-16", "source": "https://github.com/Morrowmake/glm53-flash-cmp170hx-recipe", "files": "", "model_family": "GLM 5.3 Flash", "engine_family": "vLLM", "decode_single": "442.1", "decode_agg": "835", "agg_users": "8", "prefill_prompt": "", "context_tokens": "262144", "p2p": "optional", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "180", "spec_decode": "DFlash2, up to 7 draft tokens for one user, 5 at two, 3 under load", "bench_note": "MiaAI-Lab, 400-token answers, temperature 0, code column (release 1.8.0, TP4)", "tested_on__precision": "1", "engine version": "recipe 1.8.0, engine 819146023f", "card count": "4", "decode tps": "442.1", "prefill tps": "3161", "vram per card": "64 GiB exposed, --gpu-memory-utilization 0.95", "power cap": "180 W", "unlock build": "cmpunlocker (64 GiB, 74 SMs); Morrowmake/cmpunlocker for P2P", "tested by": "voidgiraffe", "tested on": "2026-10-16", "model family": "GLM 5.3 Flash", "engine family": "vLLM", "decode single": "442.1", "decode agg": "835", "agg users": "8", "prefill prompt": "", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "180", "spec decode": "DFlash2, up to 7 draft tokens for one user, 5 at two, 3 under load", "bench note": "MiaAI-Lab, 400-token answers, temperature 0, code column (release 1.8.0, TP4)", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 Flash on 4x 8 GB (kaka86mm EXL3 + Marlin sidecar + DFlash2)", "page": "GLM 5.3 Flash on 4x 8 GB (kaka86mm EXL3 + Marlin sidecar + DFlash2)", "model": "GLM-5.3-Flash (320B MoE, 18B active, multimodal)", "quant": "EXL3 4 bpw (Mia-AiLab/GLM-5.3-Flash-EXL3-TR3-4bpw, 164 GB) plus an offline-generated Marlin INT4 decode sidecar (151 GB)", "engine": "vLLM: hyd998877/vllm_170hx_glm53-flash-exl3-optimized with kaka86mm's overlay patches; exllamav3 1.4.8", "engine_version": "final production build v30 (numbers measured on v24); repo commit 218b47f", "cards": "8 GB", "card_count": "4", "topology": "PP4 (VLLM_PP_LAYER_PARTITION=13,11,11,10), DFlash2 k=7; PCIe Gen2 x4, no P2P", "context": "524,288", "decode_tps": "96", "prefill_tps": "2077", "vram_per_card": "64 GB, --gpu-memory-utilization 0.93 (0.94 is the ceiling)", "power_cap": "200 W", "driver": "610.43.03 (open modules)", "unlock_build": "cmpunlocker (64 GB)", "flags": "--pipeline-parallel-size 4 --max-model-len 524288 --max-num-seqs 8 --max-num-batched-tokens 2048 --gpu-memory-utilization 0.93 --block-size 256 --linear-backend marlin --tool-call-parser glm47 --reasoning-parser glm45, FULL_AND_PIECEWISE CUDA graphs, DFlash2 with 7 speculative tokens and greedy drafting; env VLLM_PP_LAYER_PARTITION=13,11,11,10, VLLM_EXL3_H2D_STAGGER_S=12, VLLM_PRETEND_NO_DEEP_GEMM=1; no CPU KV tier", "status": "Reported", "tested_by": "dkpoulsen", "tested_on": "2026-09-19", "source": "https://github.com/kaka86mm/glm53-flash-170hx", "files": "deploy/build.sh, deploy/lanes2.sh, deploy/launch_exl3.sh and patches 001 to 012 (Apache-2.0; patch 006 AGPL-3.0)", "model_family": "GLM 5.3 Flash", "engine_family": "vLLM", "decode_single": "", "decode_agg": "199", "agg_users": "4", "prefill_prompt": "131k tokens", "context_tokens": "524288", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "200", "spec_decode": "DFlash2, 7 draft tokens, greedy drafting", "bench_note": "repo's own v24 table, 200 W cap; prompt and answer lengths not stated", "tested_on__precision": "1", "engine version": "final production build v30 (numbers measured on v24); repo commit 218b47f", "card count": "4", "decode tps": "96", "prefill tps": "2077", "vram per card": "64 GB, --gpu-memory-utilization 0.93 (0.94 is the ceiling)", "power cap": "200 W", "unlock build": "cmpunlocker (64 GB)", "tested by": "dkpoulsen", "tested on": "2026-09-19", "model family": "GLM 5.3 Flash", "engine family": "vLLM", "decode single": "", "decode agg": "199", "agg users": "4", "prefill prompt": "131k tokens", "context tokens": "524288", "pcie link": "Gen2", "pcie width": "x4", "power w": "200", "spec decode": "DFlash2, 7 draft tokens, greedy drafting", "bench note": "repo's own v24 table, 200 W cap; prompt and answer lengths not stated", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 Flash on 8x 8 GB (promisezackr PP8 NVFP4 DFlash2)", "page": "GLM 5.3 Flash on 8x 8 GB (promisezackr PP8 NVFP4 DFlash2)", "model": "GLM-5.3-Flash", "quant": "LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt NVFP4, routed experts only, 181 GiB), run through Marlin W4A16 kernels; GLM-5.3-Flash DFlash2 drafter (5 sliding-window layers)", "engine": "vLLM: official vllm/vllm-openai:glm53-flash image plus promisezackr's 24 patches (SM8x base from 344303947/vllm170hx, pipeline parallel for GLM-5.3-Flash, DFlash2 under PP8)", "engine_version": "v0.1.dev20051+g487ecf187 (the image's vLLM); repo commit 90ec72e", "cards": "8 GB", "card_count": "8", "topology": "PP8 (VLLM_PP_LAYER_PARTITION=7,5,7,5,7,5,7,2) with DFlash2, 7 draft tokens; PCIe Gen2 x4, no P2P; host 2x Xeon E5-2690 v4, 96 GB DDR4", "context": "1,048,576 (KV pool 6.62M tokens at utilization 0.90, about 6.3 concurrent 1M requests)", "decode_tps": "123", "prefill_tps": "", "vram_per_card": "64 GB, --gpu-memory-utilization 0.90", "power_cap": "", "driver": "", "unlock_build": "cmpunlocker VRAM unlock (64 GB per card)", "flags": "--pipeline-parallel-size 8 --max-model-len 1048576 --max-num-seqs 8 --max-num-batched-tokens 4096 --gpu-memory-utilization 0.90 --no-enable-prefix-caching --speculative-config with method dflash and num_speculative_tokens 7 --enable-auto-tool-choice --tool-call-parser glm47 --reasoning-parser glm45; env VLLM_PP_LAYER_PARTITION=7,5,7,5,7,5,7,2 VLLM_PP_MAX_DECODE_REQS_PER_BATCH=2 VLLM_GLM5N_SIDECAR_BLOCK_SIZE=256 VLLM_WORKER_MULTIPROC_METHOD=spawn CUDA_DEVICE_ORDER=PCI_BUS_ID; host: CPU governor performance, nvidia-smi -lgc 1695", "status": "Reported", "tested_by": "promisezackr", "tested_on": "2026-08-31", "source": "https://github.com/promisezackr/glm53-flash-170hx-pp8", "files": "scripts/run_prod.sh, patches 0001 to 0024 and patches/all-changes-vs-baseline.diff, apply.sh, scripts/bench.py, conc2.py, needle.py, qual.py and preflight.py (Apache-2.0)", "model_family": "GLM 5.3 Flash", "engine_family": "vLLM", "decode_single": "123", "decode_agg": "276", "agg_users": "16", "prefill_prompt": "", "context_tokens": "1048576", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "", "spec_decode": "DFlash2, 7 draft tokens", "bench_note": "bench.py, temp 0, code row; agg: conc2.py chat requests, 200-token completions", "tested_on__precision": "1", "engine version": "v0.1.dev20051+g487ecf187 (the image's vLLM); repo commit 90ec72e", "card count": "8", "decode tps": "123", "prefill tps": "", "vram per card": "64 GB, --gpu-memory-utilization 0.90", "power cap": "", "unlock build": "cmpunlocker VRAM unlock (64 GB per card)", "tested by": "promisezackr", "tested on": "2026-08-31", "model family": "GLM 5.3 Flash", "engine family": "vLLM", "decode single": "123", "decode agg": "276", "agg users": "16", "prefill prompt": "", "context tokens": "1048576", "pcie link": "Gen2", "pcie width": "x4", "power w": "", "spec decode": "DFlash2, 7 draft tokens", "bench note": "bench.py, temp 0, code row; agg: conc2.py chat requests, 200-token completions", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 on 8x 8 GB (cebeuq glm53-cmp170hx-recipe)", "page": "GLM 5.3 on 8x 8 GB (cebeuq glm53-cmp170hx-recipe)", "model": "GLM-5.3 (the full 78-layer model, not Flash): 753B total parameters (744B without the MTP draft layer)", "quant": "hybrid: the published NVFP4 routed experts unchanged plus an INT8-g128 residue, converted offline from dealignai/GLM-5.3-ABLITERATED-NVFP4 (432.9 GiB source)", "engine": "lazymio/vllm-backport sm_80 image (wtdcode/vllm-backport), pinned by digest, with whole-file vLLM patches mounted read-only", "engine_version": "vLLM g29e66dad4 in the pinned lazymio/vllm-backport image", "cards": "8 GB", "card_count": "8", "topology": "TP2 x PP4, layers 22,20,20,16, the MTP layer on the last stage; each TP2 pair behind one PCIe switch", "context": "524,288 per request, 8 sequences, KV pool 569,792 tokens", "decode_tps": "68.3", "prefill_tps": "", "vram_per_card": "64 GB (65,536 MiB), GPU memory utilization 0.97", "power_cap": "160 W per card, SM clock up to 1,695 MHz; about 2.3 kW peak at the wall", "driver": "NVIDIA open 615.71.09 with the cmpunlocker patch set plus Morrowmake's TRAP31 P2P patch", "unlock_build": "cmpunlocker 2d10f891 with Morrowmake/cmpunlocker 1f2c498f p2p-unlock.patch, ForceP2P=0x11", "flags": "'\"`UNIQ--nowiki-0000000A-QINU`\"'", "status": "Reported", "tested_by": "cebeuq", "tested_on": "2026-10-10", "source": "https://github.com/cebeuq/glm53-cmp170hx-recipe", "files": "", "model_family": "GLM 5.3", "engine_family": "vLLM", "decode_single": "68.3", "decode_agg": "135.5", "agg_users": "8", "prefill_prompt": "", "context_tokens": "524288", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "160", "spec_decode": "MTP k=2 (the model's own MTP layer, stock head) plus PipeSpec", "bench_note": "author's chat bench: 36 prompts, temperature 0, thinking on, 384 tokens, mean streaming tok/s, coding row (68.3-68.6) at 358K context / 4 sequences; aggregate = 8-stream raw-prose harness at 524K", "tested_on__precision": "1", "engine version": "vLLM g29e66dad4 in the pinned lazymio/vllm-backport image", "card count": "8", "decode tps": "68.3", "prefill tps": "", "vram per card": "64 GB (65,536 MiB), GPU memory utilization 0.97", "power cap": "160 W per card, SM clock up to 1,695 MHz; about 2.3 kW peak at the wall", "unlock build": "cmpunlocker 2d10f891 with Morrowmake/cmpunlocker 1f2c498f p2p-unlock.patch, ForceP2P=0x11", "tested by": "cebeuq", "tested on": "2026-10-10", "model family": "GLM 5.3", "engine family": "vLLM", "decode single": "68.3", "decode agg": "135.5", "agg users": "8", "prefill prompt": "", "context tokens": "524288", "pcie link": "Gen2", "pcie width": "x16", "power w": "160", "spec decode": "MTP k=2 (the model's own MTP layer, stock head) plus PipeSpec", "bench note": "author's chat bench: 36 prompts, temperature 0, thinking on, 384 tokens, mean streaming tok/s, coding row (68.3-68.6) at 358K context / 4 sequences; aggregate = 8-stream raw-prose harness at 524K", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 on 8x 8 GB (Int4-Int8Mix PP8)", "page": "GLM 5.3 on 8x 8 GB (Int4-Int8Mix PP8)", "model": "GLM-5.3 (full 78-layer, not Flash)", "quant": "GLM-5.3-Int4-Int8Mix (INT4 / INT8 mix); KV fp8_ds_mla", "engine": "vLLM (build not stated)", "engine_version": "", "cards": "8 GB", "card_count": "8", "topology": "PP8, TP1", "context": "1,048,576", "decode_tps": "25", "prefill_tps": "", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.97", "power_cap": "200 W", "driver": "", "unlock_build": "", "flags": "--pipeline-parallel-size 8 --tensor-parallel-size 1 --speculative-config mtp 5 --attention-backend TRITON_MLA_SPARSE --kv-cache-dtype fp8_ds_mla --block-size 64 --no-async-scheduling", "status": "Reported", "tested_by": "1a2b3c0967", "tested_on": "2026-10-07", "source": "https://discord.com/channels/1518634379604136178/1528826330454429706/1557523316120748073", "files": "", "model_family": "GLM 5.3", "engine_family": "vLLM", "decode_single": "25", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "1048576", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "200", "spec_decode": "MTP, 5 speculative tokens", "bench_note": "", "tested_on__precision": "1", "engine version": "", "card count": "8", "decode tps": "25", "prefill tps": "", "vram per card": "64 GiB, --gpu-memory-utilization 0.97", "power cap": "200 W", "unlock build": "", "tested by": "1a2b3c0967", "tested on": "2026-10-07", "model family": "GLM 5.3", "engine family": "vLLM", "decode single": "25", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "1048576", "pcie link": "", "pcie width": "", "power w": "200", "spec decode": "MTP, 5 speculative tokens", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 on 8x 8 GB (vllm-170hx-glm5)", "page": "GLM 5.3 on 8x 8 GB (vllm-170hx-glm5)", "model": "GLM-5.3 (full 78-layer DSA MoE, not Flash)", "quant": "GLM-5.3-Int4-Int8Mix-AWQ-g64; drafter incoai/GLM-5.3-DFlash2", "engine": "Morrowmake/vllm-cmp170hx branch ampere-glm53 plus fail-closed source patches", "engine_version": "vLLM 0.30.1rc1, fork commit 378c37b0", "cards": "8 GB", "card_count": "8", "topology": "PP8 (also PP9 and PP10), TP1", "context": "1,048,576 per request", "decode_tps": "48.7", "prefill_tps": "1833", "vram_per_card": "64 GiB, GPU_UTIL 0.96 at PP8", "power_cap": "170 W (author's 10-card run)", "driver": "", "unlock_build": "", "flags": "--attention-backend TRITON_MLA_SPARSE, packed fp8_ds_mla KV, --block-size 128, async scheduling required; PP8 partition 11,10,10,10,10,10,9,8", "status": "Proven", "tested_by": "reala7096", "tested_on": "2026-09-29", "source": "https://github.com/arvkmr/vllm-170hx-glm5", "files": "", "model_family": "GLM 5.3", "engine_family": "vLLM", "decode_single": "48.7", "decode_agg": "174.1", "agg_users": "8", "prefill_prompt": "about 7.8K tokens", "context_tokens": "1048576", "p2p": "", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "170", "spec_decode": "DFlash2 (draft length not stated; 3.92 accepted tokens per step)", "bench_note": "author's 10-card PP10 run, 170 W, MAX_SEQS 8; answer length not stated", "tested_on__precision": "1", "engine version": "vLLM 0.30.1rc1, fork commit 378c37b0", "card count": "8", "decode tps": "48.7", "prefill tps": "1833", "vram per card": "64 GiB, GPU_UTIL 0.96 at PP8", "power cap": "170 W (author's 10-card run)", "unlock build": "", "tested by": "reala7096", "tested on": "2026-09-29", "model family": "GLM 5.3", "engine family": "vLLM", "decode single": "48.7", "decode agg": "174.1", "agg users": "8", "prefill prompt": "about 7.8K tokens", "context tokens": "1048576", "pcie link": "Gen2", "pcie width": "x4", "power w": "170", "spec decode": "DFlash2 (draft length not stated; 3.92 accepted tokens per step)", "bench note": "author's 10-card PP10 run, 170 W, MAX_SEQS 8; answer length not stated", "tested on__precision": "1"}, {"pagetitle": "GLM 5.3 on 8x 8 GB plus 2x RTX PRO 6000 (vLLM TP2 x PP5)", "page": "GLM 5.3 on 8x 8 GB plus 2x RTX PRO 6000 (vLLM TP2 x PP5)", "model": "GLM-5.3 (the full 78-layer model, not Flash): 753B total parameters (744B without the MTP draft layer)", "quant": "cebeuq's hybrid: the published NVFP4 routed experts unchanged plus an INT8-g128 residue, converted from dealignai/GLM-5.3-ABLITERATED-NVFP4; about 420 GB on disk", "engine": "cebeuq's glm53-cmp170hx-recipe (patches on lazymio/vllm-backport) plus the lab's additions; his pinned sm_80 image on the CMPs, a lab sm_120 build for the Blackwells", "engine_version": "", "cards": "8 GB", "card_count": "8", "topology": "TP2 x PP5 over two hosts: stage 0 = two RTX PRO 6000 Blackwell (layers 0-29 plus embedding), stages 1-4 = four CMP 170HX pairs (12 layers each); direct 100 GbE between the hosts", "context": "524,288 per request, 8 sequences, KV pool 817,024 tokens", "decode_tps": "104", "prefill_tps": "3400", "vram_per_card": "64 GB unlocked; per CMP about 32.5 GiB loaded and 28.6 GiB for KV (last pair 43.7 and 17.3)", "power_cap": "", "driver": "", "unlock_build": "", "flags": "vLLM multiprocess multi-node executor, NCCL over IB verbs on the 100 GbE link; from cebeuq's recipe: MTP with 2 draft tokens plus PipeSpec, int8-tile latent KV (fp8_ds_mla layout), custom PCIe all-reduce inside each CMP pair", "status": "Reported", "tested_by": "darkmaniac7", "tested_on": "2026-11-05", "source": "https://github.com/cebeuq/glm53-cmp170hx-recipe", "files": "", "model_family": "GLM 5.3", "engine_family": "vLLM", "decode_single": "104", "decode_agg": "244", "agg_users": "8", "prefill_prompt": "17k tokens, cold, no prefix cache", "context_tokens": "524288", "p2p": "on", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "", "spec_decode": "MTP k=2 (the model's own MTP layer) with PipeSpec, from cebeuq's recipe; the lab's probabilistic drafts for sampled requests", "bench_note": "lab bench at the engine port: 1,024-token answers, median of 3 runs, temperature 0, code row; aggregate = 8 streams, code", "tested_on__precision": "1", "engine version": "", "card count": "8", "decode tps": "104", "prefill tps": "3400", "vram per card": "64 GB unlocked; per CMP about 32.5 GiB loaded and 28.6 GiB for KV (last pair 43.7 and 17.3)", "power cap": "", "unlock build": "", "tested by": "darkmaniac7", "tested on": "2026-11-05", "model family": "GLM 5.3", "engine family": "vLLM", "decode single": "104", "decode agg": "244", "agg users": "8", "prefill prompt": "17k tokens, cold, no prefix cache", "context tokens": "524288", "pcie link": "Gen2", "pcie width": "x16", "power w": "", "spec decode": "MTP k=2 (the model's own MTP layer) with PipeSpec, from cebeuq's recipe; the lab's probabilistic drafts for sampled requests", "bench note": "lab bench at the engine port: 1,024-token answers, median of 3 runs, temperature 0, code row; aggregate = 8 streams, code", "tested on__precision": "1"}, {"pagetitle": "Laguna-S-2.1 on 2x 10 GB", "page": "Laguna-S-2.1 on 2x 10 GB", "model": "Laguna-S-2.1, jcbtc/Laguna-S-2.1-NVFP4-GGUF", "quant": "NVFP4 GGUF (runs through a fallback path: the cards have no native FP4)", "engine": "custom llama-server backend from ciruinference (the service is named vllm, but it is not vLLM)", "engine_version": "", "cards": "10 GB", "card_count": "2", "topology": "tensor split 1,1 across the two cards", "context": "131,072", "decode_tps": "53", "prefill_tps": "", "vram_per_card": "", "power_cap": "", "driver": "", "unlock_build": "", "flags": "--ctx-size 131072 --parallel 30 --batch-size 512 --ubatch-size 256 --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0 --tensor-split 1,1 --spec-type none --no-context-shift", "status": "Reported", "tested_by": "squishysquare", "tested_on": "2026-08-12", "source": "https://discord.com/channels/1518634379604136178/1518692084729511967/1536891234214223925", "files": "", "model_family": "Laguna-S-2.1", "engine_family": "llama.cpp", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "131072", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "none", "bench_note": "one post: 53 tok/s decode, number of users not stated (the server has 30 slots)", "tested_on__precision": "1", "engine version": "", "card count": "2", "decode tps": "53", "prefill tps": "", "vram per card": "", "power cap": "", "unlock build": "", "tested by": "squishysquare", "tested on": "2026-08-12", "model family": "Laguna-S-2.1", "engine family": "llama.cpp", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "131072", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "none", "bench note": "one post: 53 tok/s decode, number of users not stated (the server has 30 slots)", "tested on__precision": "1"}, {"pagetitle": "MiMo V2.6 Flash on 4x 8 GB", "page": "MiMo V2.6 Flash on 4x 8 GB", "model": "MiMo-V2.6-Flash-RL (309B MoE, 15B active; text, image, video and audio input)", "quant": "NVFP4 (ModelOpt mixed precision): ProCreations/MiMo-V2.6-Flash-RL-NVFP4, run as 4-bit weights with BF16 activations through Marlin", "engine": "Morrowmake/vllm-cmp170hx image (tag qwen38-recipe-1.0.0) plus one mounted file (vLLM #58235 vision-sink fix)", "engine_version": "recipe 0.2.1; engine 18305948ac", "cards": "8 GB", "card_count": "4", "topology": "TP4 (P2P on); expert parallel off", "context": "1,048,576", "decode_tps": "274.9", "prefill_tps": "2722", "vram_per_card": "64 GiB; weights 45.5 GiB plus 10 GiB KV per card; GPU_UTIL 0.94 (fixed)", "power_cap": "180 W", "driver": "580 or newer (measured on 610.57.04)", "unlock_build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable build", "flags": "--tensor-parallel-size 4 --kv-cache-dtype bfloat16 --block-size 128 --max-model-len 1048576 --gpu-memory-utilization 0.94 --kv-cache-memory 10737418240 --max-num-seqs 32 --max-num-batched-tokens 2048; DFlash 7/5/3 by load", "status": "Reported", "tested_by": "nathang6866", "tested_on": "2026-10-18", "source": "https://github.com/ElectroHobo/mimo-v26-flash-cmp170hx-recipe", "files": "", "model_family": "MiMo V2.6 Flash", "engine_family": "vLLM", "decode_single": "274.9", "decode_agg": "738", "agg_users": "8", "prefill_prompt": "24K to 38K tokens", "context_tokens": "1048576", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "180", "spec_decode": "DFlash (MiMo drafter), 7 draft tokens at 1 request, 5 at 2, 3 from 3", "bench_note": "MiaAI-Lab, 400-token replies, temperature 0, thinking off, code column", "tested_on__precision": "1", "engine version": "recipe 0.2.1; engine 18305948ac", "card count": "4", "decode tps": "274.9", "prefill tps": "2722", "vram per card": "64 GiB; weights 45.5 GiB plus 10 GiB KV per card; GPU_UTIL 0.94 (fixed)", "power cap": "180 W", "unlock build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable build", "tested by": "nathang6866", "tested on": "2026-10-18", "model family": "MiMo V2.6 Flash", "engine family": "vLLM", "decode single": "274.9", "decode agg": "738", "agg users": "8", "prefill prompt": "24K to 38K tokens", "context tokens": "1048576", "pcie link": "Gen2", "pcie width": "x16", "power w": "180", "spec decode": "DFlash (MiMo drafter), 7 draft tokens at 1 request, 5 at 2, 3 from 3", "bench note": "MiaAI-Lab, 400-token replies, temperature 0, thinking off, code column", "tested on__precision": "1"}, {"pagetitle": "Mistral Large 4 on 8x 8 GB (llama.cpp lab trial)", "page": "Mistral Large 4 on 8x 8 GB (llama.cpp lab trial)", "model": "Mistral-Large-4-1T-A52B (\"Le Chonk\")", "quant": "unsloth UD-IQ3_XXS GGUF (394 GiB in 10 files); DSpark drafter Q8_0 (2.1 GB)", "engine": "llama.cpp llama-server, built in a CUDA 12.9 container", "engine_version": "llama.cpp master a91eab6 (2026-10-31), model support from PR #31376", "cards": "8 GB", "card_count": "8", "topology": "Layer split over all eight cards (tensor split is not available for this architecture)", "context": "196,608 (one unified KV pool, 8 slots)", "decode_tps": "18.7", "prefill_tps": "131", "vram_per_card": "54 to 63 GiB of 64 used", "power_cap": "250 to 300 W (governor), +160 MHz offset", "driver": "615.71.09 with cmpunlocker", "unlock_build": "cmpunlocker (64 GiB, 74 SMs)", "flags": "-ngl 999 -sm layer -c 196608 -np 8 -kvu --spec-type draft-dspark -md (DSpark Q8_0) --reasoning-format deepseek; built with GGML_CUDA_FORCE_CUBLAS", "status": "Reported", "tested_by": "darkmaniac7", "tested_on": "2026-11-01", "source": "https://huggingface.co/unsloth/Mistral-Large-4-1T-A52B-GGUF", "files": "", "model_family": "Mistral Large 4", "engine_family": "llama.cpp", "decode_single": "18.7", "decode_agg": "34.1", "agg_users": "8", "prefill_prompt": "17k tokens", "context_tokens": "196608", "p2p": "on", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "DSpark (Q8_0 drafter), 3 draft tokens per step", "bench_note": "lab bench, short prompts, 1,024-token replies, reasoning effort low, code row", "tested_on__precision": "1", "engine version": "llama.cpp master a91eab6 (2026-10-31), model support from PR #31376", "card count": "8", "decode tps": "18.7", "prefill tps": "131", "vram per card": "54 to 63 GiB of 64 used", "power cap": "250 to 300 W (governor), +160 MHz offset", "unlock build": "cmpunlocker (64 GiB, 74 SMs)", "tested by": "darkmaniac7", "tested on": "2026-11-01", "model family": "Mistral Large 4", "engine family": "llama.cpp", "decode single": "18.7", "decode agg": "34.1", "agg users": "8", "prefill prompt": "17k tokens", "context tokens": "196608", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "DSpark (Q8_0 drafter), 3 draft tokens per step", "bench note": "lab bench, short prompts, 1,024-token replies, reasoning effort low, code row", "tested on__precision": "1"}, {"pagetitle": "Qwen3.6 27B on 1x 10 GB (llama.cpp MTP)", "page": "Qwen3.6 27B on 1x 10 GB (llama.cpp MTP)", "model": "Qwen3.6-27B (dense), unsloth/Qwen3.6-27B-MTP-GGUF", "quant": "UD-Q4_K_XL GGUF (UD-Q8_K_XL also tested)", "engine": "llama.cpp llama-server with draft MTP (ik_llama.cpp also tested)", "engine_version": "llama.cpp 10095 (e8e6c7af2); ik_llama.cpp 4735 (9d07d868)", "cards": "10 GB", "card_count": "1", "topology": "single card, whole model in VRAM", "context": "", "decode_tps": "46.24", "prefill_tps": "496.85", "vram_per_card": "40 GB profile; 17,464 MiB used at Q4, 35,718 MiB at Q8", "power_cap": "250 W maximum", "driver": "", "unlock_build": "cmpunlocker (10 GB to 40 GB)", "flags": "llama.cpp: --spec-type draft-mtp --spec-draft-n-max 2; ik_llama.cpp: --spec-type mtp:n_max=2,p_min=0.0; both built with Zen 3 and CC 8.0 flags", "status": "Reported", "tested_by": "eksitus", "tested_on": "2026-07-22", "source": "https://discord.com/channels/1518634379604136178/1528826330454429706/1529554808749690921", "files": "", "model_family": "Qwen3.6 27B", "engine_family": "llama.cpp", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "2048 tokens", "context_tokens": "", "p2p": "off", "pcie_link": "Gen1", "pcie_width": "x4", "power_w": "250", "spec_decode": "MTP, draft-mtp with 2 draft tokens", "bench_note": "tg128 and pp2048, Q4 on llama.cpp with draft MTP (tool and users not stated)", "tested_on__precision": "1", "engine version": "llama.cpp 10095 (e8e6c7af2); ik_llama.cpp 4735 (9d07d868)", "card count": "1", "decode tps": "46.24", "prefill tps": "496.85", "vram per card": "40 GB profile; 17,464 MiB used at Q4, 35,718 MiB at Q8", "power cap": "250 W maximum", "unlock build": "cmpunlocker (10 GB to 40 GB)", "tested by": "eksitus", "tested on": "2026-07-22", "model family": "Qwen3.6 27B", "engine family": "llama.cpp", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "2048 tokens", "context tokens": "", "pcie link": "Gen1", "pcie width": "x4", "power w": "250", "spec decode": "MTP, draft-mtp with 2 draft tokens", "bench note": "tg128 and pp2048, Q4 on llama.cpp with draft MTP (tool and users not stated)", "tested on__precision": "1"}, {"pagetitle": "Qwen3.6 27B on 1x 10 GB (SGLang W8A8)", "page": "Qwen3.6 27B on 1x 10 GB (SGLang W8A8)", "model": "Qwen3.6-27B, Avesed/Qwen3.6-27B-INT8-W8A8", "quant": "INT8 W8A8 (compressed-tensors, dynamic)", "engine": "SGLang", "engine_version": "0.5.16 (fdebc938); Triton 3.6.0, FlashInfer 0.6.14, PyTorch 2.11.0+cu130", "cards": "10 GB", "card_count": "1", "topology": "single card, in a KVM virtual machine", "context": "24,576 tokens (the total token pool at one request; 20,480 at four)", "decode_tps": "90.26", "prefill_tps": "4122.16", "vram_per_card": "40 GiB profile (40,960 MiB)", "power_cap": "250 W", "driver": "610.43.02 (CUDA 13.3.73)", "unlock_build": "", "flags": "MTP through EAGLE, 3 steps, top-k 1, 4 draft tokens; page size 64; chunked prefill 2,048 tokens; static memory fraction 0.94", "status": "Reported", "tested_by": "", "tested_on": "2026-07-31", "source": "https://github.com/Ciender/CMP-170HX-SGLang-Qwen3.6-Benchmarks", "files": "", "model_family": "Qwen3.6 27B", "engine_family": "SGLang", "decode_single": "90.26", "decode_agg": "269.31", "agg_users": "4", "prefill_prompt": "2048 tokens", "context_tokens": "24576", "p2p": "off", "pcie_link": "Gen1", "pcie_width": "x4", "power_w": "250", "spec_decode": "MTP (EAGLE), 3 steps, 4 draft tokens", "bench_note": "SGLang serving bench, temperature 0; 1 user: 1,024 in, 8,192 out; 4 users: 2,048 in and out", "tested_on__precision": "1", "engine version": "0.5.16 (fdebc938); Triton 3.6.0, FlashInfer 0.6.14, PyTorch 2.11.0+cu130", "card count": "1", "decode tps": "90.26", "prefill tps": "4122.16", "vram per card": "40 GiB profile (40,960 MiB)", "power cap": "250 W", "unlock build": "", "tested by": "", "tested on": "2026-07-31", "model family": "Qwen3.6 27B", "engine family": "SGLang", "decode single": "90.26", "decode agg": "269.31", "agg users": "4", "prefill prompt": "2048 tokens", "context tokens": "24576", "pcie link": "Gen1", "pcie width": "x4", "power w": "250", "spec decode": "MTP (EAGLE), 3 steps, 4 draft tokens", "bench note": "SGLang serving bench, temperature 0; 1 user: 1,024 in, 8,192 out; 4 users: 2,048 in and out", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 27B on 1x 8 GB (W4A16 vLLM)", "page": "Qwen3.8 27B on 1x 8 GB (W4A16 vLLM)", "model": "Qwen3.8-27B (dense)", "quant": "W4A16 (INT4 weights, Marlin kernels)", "engine": "vLLM", "engine_version": "", "cards": "8 GB", "card_count": "1", "topology": "single card", "context": "262,144 native (YaRN to about 500k in the recipe's long variant)", "decode_tps": "169.5", "prefill_tps": "", "vram_per_card": "64 GB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "", "status": "Proven", "tested_by": "wakeupmbrc1", "tested_on": "2026-08-30", "source": "https://www.reddit.com/r/LocalLLM/comments/1w2f03e/qwen3827b_w4a16_on_one_64_gb_cmp_170hx_16070/", "files": "", "model_family": "Qwen3.8 27B", "engine_family": "vLLM", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "262144", "p2p": "off", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "DFlash2", "bench_note": "", "tested_on__precision": "1", "engine version": "", "card count": "1", "decode tps": "169.5", "prefill tps": "", "vram per card": "64 GB", "power cap": "", "unlock build": "", "tested by": "wakeupmbrc1", "tested on": "2026-08-30", "model family": "Qwen3.8 27B", "engine family": "vLLM", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "262144", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "DFlash2", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 1x 8 GB (ElectroHobo 3 bpw)", "page": "Qwen3.8 Flash Next on 1x 8 GB (ElectroHobo 3 bpw)", "model": "Qwen3.8-Flash-Next (125B MoE per the recipe)", "quant": "AutoRound mixed 2/3-bit (klee100, 2.998 effective bpw, 47 GiB on card) plus halt95's FP8 n-gram table in host RAM", "engine": "Morrowmake/vllm-cmp170hx image with patch files mounted read-only", "engine_version": "recipe 0.2.1; engine 18305948ac", "cards": "8 GB", "card_count": "1", "topology": "single card", "context": "262,144", "decode_tps": "296.4", "prefill_tps": "4243", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.95 (fixed)", "power_cap": "180 W", "driver": "580 or newer (measured on 610.57.04)", "unlock_build": "cmpunlocker (64 GiB, 74 SMs)", "flags": "'\"`UNIQ--nowiki-0000005C-QINU`\"'", "status": "Reported", "tested_by": "nathang6866", "tested_on": "2026-10-23", "source": "https://github.com/ElectroHobo/qwen38-flash-next-3bpw-1x-cmp170hx-recipe", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "296.4", "decode_agg": "1384", "agg_users": "48", "prefill_prompt": "", "context_tokens": "262144", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "180", "spec_decode": "DFlash2, 7 draft tokens (1 to 2 requests; 5 for 3 to 4, 3 beyond)", "bench_note": "MiaAI-Lab protocol, code column (structured / code / prose)", "tested_on__precision": "1", "engine version": "recipe 0.2.1; engine 18305948ac", "card count": "1", "decode tps": "296.4", "prefill tps": "4243", "vram per card": "64 GiB, --gpu-memory-utilization 0.95 (fixed)", "power cap": "180 W", "unlock build": "cmpunlocker (64 GiB, 74 SMs)", "tested by": "nathang6866", "tested on": "2026-10-23", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "296.4", "decode agg": "1384", "agg users": "48", "prefill prompt": "", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "180", "spec decode": "DFlash2, 7 draft tokens (1 to 2 requests; 5 for 3 to 4, 3 beyond)", "bench note": "MiaAI-Lab protocol, code column (structured / code / prose)", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 1x 8 GB (iIIusi0n SSD PLE)", "page": "Qwen3.8 Flash Next on 1x 8 GB (iIIusi0n SSD PLE)", "model": "Qwen3.8-Flash-Next", "quant": "AutoRound 3 bpw + MTP: klee100/Qwen3.8-Flash-Next-AutoRound-3bpw-MTP; 95.37 GiB BF16 PLE table on SSD", "engine": "vLLM a5a30471 plus a 13-file downstream patch (native Linux AIO PLE reader)", "engine_version": "0.29.1rc1.dev402+ga5a30471f.ple1", "cards": "8 GB", "card_count": "1", "topology": "single card", "context": "262,144 advertised (tested through 32K)", "decode_tps": "110.99", "prefill_tps": "2636", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.96", "power_cap": "180 W", "driver": "610.43.03", "unlock_build": "cmpunlocker (64 GiB)", "flags": "--max-num-seqs 16 --max-num-batched-tokens 2048 --gpu-memory-utilization 0.96 --mamba-cache-mode align --enable-prefix-caching cudagraph FULL_AND_PIECEWISE; MTP 1; ple_ssd_offload true, ple_ssd_io_depth 256, ple_ssd_cache_mb 512", "status": "Proven", "tested_by": "iIIusi0n", "tested_on": "2026-09-19", "source": "https://github.com/iIIusi0n/qwen3.8-flash-next-cmp170hx", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "110.99", "decode_agg": "667", "agg_users": "16", "prefill_prompt": "8K tokens", "context_tokens": "262144", "p2p": "off", "pcie_link": "", "pcie_width": "", "power_w": "180", "spec_decode": "MTP 1", "bench_note": "three-round medians, 128 output tokens, temperature 0, thinking off", "tested_on__precision": "1", "engine version": "0.29.1rc1.dev402+ga5a30471f.ple1", "card count": "1", "decode tps": "110.99", "prefill tps": "2636", "vram per card": "64 GiB, --gpu-memory-utilization 0.96", "power cap": "180 W", "unlock build": "cmpunlocker (64 GiB)", "tested by": "iIIusi0n", "tested on": "2026-09-19", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "110.99", "decode agg": "667", "agg users": "16", "prefill prompt": "8K tokens", "context tokens": "262144", "pcie link": "", "pcie width": "", "power w": "180", "spec decode": "MTP 1", "bench note": "three-round medians, 128 output tokens, temperature 0, thinking off", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (DUFRelic PP2)", "page": "Qwen3.8 Flash Next on 2x 8 GB (DUFRelic PP2)", "model": "Qwen3.8-Flash-Next", "quant": "W4A16 INT4 g128 with the PLE table repacked INT4 g32: zankich/Qwen3.8-Flash-Next-W4A16-Merlin-INT4PLE-FP8KV; KV BF16", "engine": "vLLM v0.30.0 + zankich INT4-PLE plugin + six hash-guarded patch layers", "engine_version": "vLLM v0.30.0 (ced6857a); plugin zankich/vllm v0.29.0z (6609bfef)", "cards": "8 GB", "card_count": "2", "topology": "PP2, MTP depth 3", "context": "262,144", "decode_tps": "133.39", "prefill_tps": "6213.66", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.90", "power_cap": "140 W (200 W observed-draw abort guard)", "driver": "NVIDIA 610.43.02", "unlock_build": "amoghmunikote/cmpunlocker 6c442eeb (patch 0016 SM reconfig, 74 SMs) plus Bayley P2P patches 0011/0013/0015", "flags": "'\"`UNIQ--nowiki-000006EF-QINU`\"'", "status": "Reported", "tested_by": "dufrelic", "tested_on": "2026-10-03", "source": "https://github.com/DUFRelic/qwen38-flash-next-recipe", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "133.39", "decode_agg": "245.17", "agg_users": "4", "prefill_prompt": "16384 tokens", "context_tokens": "262144", "p2p": "optional", "pcie_link": "", "pcie_width": "", "power_w": "140", "spec_decode": "MTP 3", "bench_note": "llama-benchy --pp 16384 --tg 2048, concurrency 1 and 4, 3 runs, no cache", "tested_on__precision": "1", "engine version": "vLLM v0.30.0 (ced6857a); plugin zankich/vllm v0.29.0z (6609bfef)", "card count": "2", "decode tps": "133.39", "prefill tps": "6213.66", "vram per card": "64 GiB, --gpu-memory-utilization 0.90", "power cap": "140 W (200 W observed-draw abort guard)", "unlock build": "amoghmunikote/cmpunlocker 6c442eeb (patch 0016 SM reconfig, 74 SMs) plus Bayley P2P patches 0011/0013/0015", "tested by": "dufrelic", "tested on": "2026-10-03", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "133.39", "decode agg": "245.17", "agg users": "4", "prefill prompt": "16384 tokens", "context tokens": "262144", "pcie link": "", "pcie width": "", "power w": "140", "spec decode": "MTP 3", "bench note": "llama-benchy --pp 16384 --tg 2048, concurrency 1 and 4, 3 runs, no cache", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (ElectroHobo W4A16 TP2)", "page": "Qwen3.8 Flash Next on 2x 8 GB (ElectroHobo W4A16 TP2)", "model": "Qwen3.8-Flash-Next (125B MoE per the recipe)", "quant": "W4A16: halt95/Qwen3.8-Flash-Next-W4A16-Merlin (AutoRound INT4 g128 experts, INT8 GDN projections, FP8 n-gram table)", "engine": "Morrowmake/vllm-cmp170hx image (Qwen EXL3 release) with patch files mounted read-only", "engine_version": "recipe 0.3.1; engine 18305948ac", "cards": "8 GB", "card_count": "2", "topology": "TP2 with whole-expert expert parallel", "context": "262,144", "decode_tps": "349.7", "prefill_tps": "4696", "vram_per_card": "64 GiB, GPU_UTIL 0.95 (fixed)", "power_cap": "180 W", "driver": "580 or newer (measured on 610.57.04); open modules with HMM for the table-on-disk mode", "unlock_build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable build", "flags": "--tensor-parallel-size 2 --enable-expert-parallel --kv-cache-dtype bfloat16 --engram-config cpu_offload --max-num-seqs 128 --block-size 912 cudagraph FULL_AND_PIECEWISE; DFlash 2 drafter 7/5/3/2 by load", "status": "Proven", "tested_by": "nathang6866", "tested_on": "2026-10-23", "source": "https://github.com/ElectroHobo/qwen38-flash-next-w4a16-2x-cmp170hx-recipe", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "349.7", "decode_agg": "1937", "agg_users": "128", "prefill_prompt": "", "context_tokens": "262144", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "x16", "power_w": "180", "spec_decode": "DFlash 2, 7/5/3/2 draft tokens by load", "bench_note": "MiaAI-Lab protocol, release 0.2.0, code column; 128-user figure at 700 tokens", "tested_on__precision": "1", "engine version": "recipe 0.3.1; engine 18305948ac", "card count": "2", "decode tps": "349.7", "prefill tps": "4696", "vram per card": "64 GiB, GPU_UTIL 0.95 (fixed)", "power cap": "180 W", "unlock build": "cmpunlocker (64 GiB, 74 SMs) with a P2P-capable build", "tested by": "nathang6866", "tested on": "2026-10-23", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "349.7", "decode agg": "1937", "agg users": "128", "prefill prompt": "", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x16", "power w": "180", "spec decode": "DFlash 2, 7/5/3/2 draft tokens by load", "bench note": "MiaAI-Lab protocol, release 0.2.0, code column; 128-user figure at 700 tokens", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (mannydelrio PP2)", "page": "Qwen3.8 Flash Next on 2x 8 GB (mannydelrio PP2)", "model": "Qwen3.8-Flash-Next-Uncensored (refusal-removed, from orcarouter)", "quant": "AWQ W4A16 g32 (leoncca/Qwen3.8-Flash-Next-Uncensored-AWQ-g32); PLE table converted to INT4 and served from SSD", "engine": "wtdcode/vllm-backport 3142ded1 plus a two-file patch and two tinybeast PLE plugins (venv, no container)", "engine_version": "vllm-backport 3142ded1", "cards": "8 GB", "card_count": "2", "topology": "PP2 (layer partition 25,23), no P2P", "context": "262,144", "decode_tps": "177.8", "prefill_tps": "7006", "vram_per_card": "64 GB, --gpu-memory-utilization 0.94, TB_TORCH_MEM_CAP_GIB=61.5", "power_cap": "180 W", "driver": "nvidia-open 610.43.02 + cmpunlocker v0.5 or newer", "unlock_build": "cmpunlocker v0.5+ (sudo ./install.sh --profile=8gb --no-iommu)", "flags": "'\"`UNIQ--nowiki-000007AD-QINU`\"'", "status": "Reported", "tested_by": "mrmannyatx", "tested_on": "2026-10-15", "source": "https://github.com/mannydelrio/qwen38-flash-next-2x-cmp170hx", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "177.8", "decode_agg": "734.2", "agg_users": "8", "prefill_prompt": "24K to 38K tokens", "context_tokens": "262144", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "180", "spec_decode": "MTP 3", "bench_note": "MiaAI-Lab prompts, temperature 0, thinking off, 400-token cap, code column", "tested_on__precision": "1", "engine version": "vllm-backport 3142ded1", "card count": "2", "decode tps": "177.8", "prefill tps": "7006", "vram per card": "64 GB, --gpu-memory-utilization 0.94, TB_TORCH_MEM_CAP_GIB=61.5", "power cap": "180 W", "unlock build": "cmpunlocker v0.5+ (sudo ./install.sh --profile=8gb --no-iommu)", "tested by": "mrmannyatx", "tested on": "2026-10-15", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "177.8", "decode agg": "734.2", "agg users": "8", "prefill prompt": "24K to 38K tokens", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x4", "power w": "180", "spec decode": "MTP 3", "bench note": "MiaAI-Lab prompts, temperature 0, thinking off, 400-token cap, code column", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (Morrowmake EXL3)", "page": "Qwen3.8 Flash Next on 2x 8 GB (Morrowmake EXL3)", "model": "Qwen3.8-Flash-Next", "quant": "EXL3 5.05 bpw; PLE (n-gram) table read from SSD", "engine": "Morrowmake/vllm-cmp170hx (EXL3 path)", "engine_version": "tag qwen38-recipe-1.0.0, engine 18305948ac", "cards": "8 GB", "card_count": "2", "topology": "TP2 with expert parallel", "context": "262,144", "decode_tps": "152.6", "prefill_tps": "2651", "vram_per_card": "64 GiB, --gpu-memory-utilization 0.95", "power_cap": "", "driver": "", "unlock_build": "", "flags": "--quantization exl3 --load-format exl3 --tensor-parallel-size 2 --enable-expert-parallel --engram-config cpu_offload false, VLLM_EXL3_PLE_PLACEMENT=ssd, cudagraph PIECEWISE, MTP 3", "status": "Reported", "tested_by": "voidgiraffe", "tested_on": "2026-10-16", "source": "https://github.com/Morrowmake/qwen38-flash-next-cmp170hx-recipe", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "152.6", "decode_agg": "674.7", "agg_users": "8", "prefill_prompt": "", "context_tokens": "262144", "p2p": "required", "pcie_link": "", "pcie_width": "x16", "power_w": "", "spec_decode": "MTP 3", "bench_note": "MiaAI-Lab protocol (per Benchmarks), Morrowmake's published figures, code column", "tested_on__precision": "1", "engine version": "tag qwen38-recipe-1.0.0, engine 18305948ac", "card count": "2", "decode tps": "152.6", "prefill tps": "2651", "vram per card": "64 GiB, --gpu-memory-utilization 0.95", "power cap": "", "unlock build": "", "tested by": "voidgiraffe", "tested on": "2026-10-16", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "152.6", "decode agg": "674.7", "agg users": "8", "prefill prompt": "", "context tokens": "262144", "pcie link": "", "pcie width": "x16", "power w": "", "spec decode": "MTP 3", "bench note": "MiaAI-Lab protocol (per Benchmarks), Morrowmake's published figures, code column", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (Plean PP2)", "page": "Qwen3.8 Flash Next on 2x 8 GB (Plean PP2)", "model": "Qwen3.8-Flash-Next", "quant": "Mixed INT4 AutoRound (Minachist/Qwen3.8-Flash-Next-INT4-Mixed-AutoRound, about 65 GB) with the full BF16 PLE table on NVMe", "engine": "vLLM 0.29.1rc1 nightly plus the checkpoint author's patches", "engine_version": "", "cards": "8 GB", "card_count": "2", "topology": "PP2, MTP 3, no P2P", "context": "262,144 (256K in the post)", "decode_tps": "197", "prefill_tps": "5800", "vram_per_card": "64 GB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "--pipeline-parallel-size 2, MTP 3 draft tokens, max concurrency 6, BF16 PLE table offloaded to NVMe", "status": "Proven", "tested_by": "plean.", "tested_on": "2026-10-07", "source": "https://www.bozocentral.com/2x-cmp170hx-qwen3-flash-next", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "262144", "p2p": "off", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "MTP 3", "bench_note": "", "tested_on__precision": "1", "engine version": "", "card count": "2", "decode tps": "197", "prefill tps": "5800", "vram per card": "64 GB", "power cap": "", "unlock build": "", "tested by": "plean.", "tested on": "2026-10-07", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "262144", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "MTP 3", "bench note": "", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (Qwen-Flash-SM80, GDS TEP2)", "page": "Qwen3.8 Flash Next on 2x 8 GB (Qwen-Flash-SM80, GDS TEP2)", "model": "Qwen3.8-Flash-Next (nvidia/Qwen3.8-Flash-Next-NVFP4, revision fc694b54, 11 shards, 132.68 GB)", "quant": "NVFP4 main weights through Marlin (no native FP4 on sm_80), mixed FP8 and BF16 MTP weights, FP8 PLE table (about 51.2 GB) kept on NVMe", "engine": "Qwen-Flash-SM80 overlay on vllm/vllm-openai:qwen38-flash-next (digest-pinned), FlashInfer PCIe IPC AllReduce, GDS/cuFile over NVMe P2PDMA", "engine_version": "v0.3.0 (commit 4d229f1, 2026-10-14); base vLLM 0.1.dev20073+g8e685d198", "cards": "8 GB", "card_count": "2", "topology": "TEP2 (TP2 plus expert parallel), MTP 6; one card at PCIe Gen2 x16 (x16 mod), one at Gen2 x4; GPU P2P required", "context": "262,144", "decode_tps": "168.29", "prefill_tps": "1986.38", "vram_per_card": "64 GB, --gpu-memory-utilization 0.93 (v0.3.0)", "power_cap": "", "driver": "NVIDIA open 610.43.03, CUDA 13.0", "unlock_build": "bayley/cmpunlocker 5a7bb4b, sudo ./install.sh --p2p --no-iommu", "flags": "--tensor-parallel-size 2 --enable-expert-parallel --quantization modelopt --linear-backend marlin --moe-backend marlin --dtype bfloat16 --max-model-len 262144 --block-size 3456 --enable-prefix-caching --mamba-cache-mode align --enable-chunked-prefill --max-num-batched-tokens 8192 --no-async-scheduling, MTP 6 (greedy draft, standard rejection); env VLLM_PLE_GDS=1 CUFILE_USE_PCIP2PDMA=1; nvidia module RMForceStaticBar1=1;RMPcieP2PType=1", "status": "Reported", "tested_by": "nguyenthimy2022kg-alt", "tested_on": "2026-09-08", "source": "https://github.com/nguyenthimy2022kg-alt/Qwen-Flash-SM80-170HX", "files": "config/vllm-args.json, config/runtime-env.json, config/example.json, scripts/serve.py and the data preparation scripts (Apache-2.0); about 24 MB of precompiled Triton kernels for the reference stack (in the repository, not mirrored here)", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "168.29", "decode_agg": "749.67", "agg_users": "10", "prefill_prompt": "8K tokens", "context_tokens": "262144", "p2p": "required", "pcie_link": "Gen2", "pcie_width": "mixed", "power_w": "", "spec_decode": "MTP 6", "bench_note": "1 req: 8K input, thinking on (xhigh); 10 req x 5,000 output tokens, thinking off, temp 1", "tested_on__precision": "1", "engine version": "v0.3.0 (commit 4d229f1, 2026-10-14); base vLLM 0.1.dev20073+g8e685d198", "card count": "2", "decode tps": "168.29", "prefill tps": "1986.38", "vram per card": "64 GB, --gpu-memory-utilization 0.93 (v0.3.0)", "power cap": "", "unlock build": "bayley/cmpunlocker 5a7bb4b, sudo ./install.sh --p2p --no-iommu", "tested by": "nguyenthimy2022kg-alt", "tested on": "2026-09-08", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "168.29", "decode agg": "749.67", "agg users": "10", "prefill prompt": "8K tokens", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "mixed", "power w": "", "spec decode": "MTP 6", "bench note": "1 req: 8K input, thinking on (xhigh); 10 req x 5,000 output tokens, thinking off, temp 1", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 2x 8 GB (vektorprime PP2)", "page": "Qwen3.8 Flash Next on 2x 8 GB (vektorprime PP2)", "model": "Qwen3.8-Flash-Next", "quant": "AWQ INT4: cyankiwi/Qwen3.8-Flash-Next-AWQ-INT4; 95 GiB BF16 PLE table in host RAM (FP8 table optional)", "engine": "vLLM nightly + PR #53899 (a5530b9) + the repo's patch scripts", "engine_version": "0.1.dev1+ga5530b90c.d20260829", "cards": "8 GB", "card_count": "2", "topology": "PP2 with PLE CPU offload", "context": "262,144", "decode_tps": "75.4", "prefill_tps": "", "vram_per_card": "64 GiB; weights PP0 39.48 GiB, PP1 45.53 GiB; --gpu-memory-utilization 0.9", "power_cap": "", "driver": "", "unlock_build": "", "flags": "--pipeline-parallel-size 2 --max-num-seqs 8 --mamba-cache-mode align --mamba-ssm-cache-dtype float32 --speculative-config mtp --enable-prefix-caching --async-scheduling; VLLM_PLE_CPU_OFFLOAD=1, VLLM_GDN_DECODE_KERNEL=triton, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", "status": "Reported", "tested_by": "vektor_prime", "tested_on": "2026-08-29", "source": "https://github.com/vektorprime/qwen38-flash-next-pp2", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "75.4", "decode_agg": "", "agg_users": "", "prefill_prompt": "", "context_tokens": "262144", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "MTP, 1 draft token (k=1)", "bench_note": "single stream, MTP k=1 (the page gives 75.4 to 75.8)", "tested_on__precision": "1", "engine version": "0.1.dev1+ga5530b90c.d20260829", "card count": "2", "decode tps": "75.4", "prefill tps": "", "vram per card": "64 GiB; weights PP0 39.48 GiB, PP1 45.53 GiB; --gpu-memory-utilization 0.9", "power cap": "", "unlock build": "", "tested by": "vektor_prime", "tested on": "2026-08-29", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "75.4", "decode agg": "", "agg users": "", "prefill prompt": "", "context tokens": "262144", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "MTP, 1 draft token (k=1)", "bench note": "single stream, MTP k=1 (the page gives 75.4 to 75.8)", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 3x 8 GB (clee PP3)", "page": "Qwen3.8 Flash Next on 3x 8 GB (clee PP3)", "model": "Qwen3.8-Flash-Next", "quant": "W4A16 AutoRound: Intel/Qwen3.8-Flash-Next-W4A16-AutoRound; 95.4 GiB BF16 PLE table streamed from NVMe; KV fp8 e4m3 (software-decoded on SM80)", "engine": "vLLM nightly 7d0b4e57 plus the author's pure-Python fork clee/vllm-sm80 (branch q38fn/stack)", "engine_version": "q38fn/stack on nightly-7d0b4e57", "cards": "8 GB", "card_count": "3", "topology": "PP3, no P2P (works at PP2 on two cards)", "context": "1,048,576 (YaRN x4)", "decode_tps": "74.4", "prefill_tps": "4831", "vram_per_card": "64 GiB, gpu-memory-utilization 0.92", "power_cap": "", "driver": "", "unlock_build": "", "flags": "pipeline-parallel-size 3, block-size 1728, kv-cache-dtype fp8_e4m3, max-model-len 1048576, max-num-seqs 8, max-num-batched-tokens 8192, FULL_AND_PIECEWISE; NCCL_P2P_DISABLE=1, VLLM_PP_LAYER_PARTITION=16,16,16, VLLM_PLE_STAGED=1", "status": "Proven", "tested_by": "c133", "tested_on": "2026-10-10", "source": "https://github.com/clee/qwen3.8-flash-next-3x170hx", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "74.4", "decode_agg": "", "agg_users": "", "prefill_prompt": "64,845 tokens", "context_tokens": "1048576", "p2p": "off", "pcie_link": "", "pcie_width": "x4", "power_w": "", "spec_decode": "none", "bench_note": "author, cold cache, single stream, temperature 0, 1,026-token prompt", "tested_on__precision": "1", "engine version": "q38fn/stack on nightly-7d0b4e57", "card count": "3", "decode tps": "74.4", "prefill tps": "4831", "vram per card": "64 GiB, gpu-memory-utilization 0.92", "power cap": "", "unlock build": "", "tested by": "c133", "tested on": "2026-10-10", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "74.4", "decode agg": "", "agg users": "", "prefill prompt": "64,845 tokens", "context tokens": "1048576", "pcie link": "", "pcie width": "x4", "power w": "", "spec decode": "none", "bench note": "author, cold cache, single stream, temperature 0, 1,026-token prompt", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 3x or 4x 8 GB (zebgop-ops FP8 PP)", "page": "Qwen3.8 Flash Next on 3x or 4x 8 GB (zebgop-ops FP8 PP)", "model": "Qwen/Qwen3.8-Flash-Next-FP8 (revision bcd9f01d)", "quant": "Official block FP8, about 126 GiB including the about 50 GiB FP8 n-gram (PLE) table, which is served from host RAM; KV cache bf16 (the QSA layers refuse fp8)", "engine": "vLLM: the vllm/vllm-openai:qwen38-flash-next image with zebgop-ops patches 0001 to 0016 (never 0008-WITHDRAWN) and three ported files", "engine_version": "0.1.dev20073+g8e685d198; repo commit 2f551bb (patch 0016, 2026-10-02)", "cards": "8 GB", "card_count": "4", "topology": "PP4, VLLM_PP_LAYER_PARTITION=12,12,12,12, MTP 3, FULL_AND_PIECEWISE CUDA graphs; PCIe Gen2 x4 per card, no P2P. Three cards: PP3 with 16,17,15 or 16,16,16", "context": "262,144 native; 524,288 with YaRN 2.0; 1,048,576 with YaRN 4.0 (needle verified on PP3)", "decode_tps": "89", "prefill_tps": "11100", "vram_per_card": "64 GiB; --gpu-memory-utilization 0.94 plus per-rank KV budgets; measured 34.0-34.3 GiB per 12-layer rank, 37.9 GiB on the drafter rank", "power_cap": "200 W", "driver": "", "unlock_build": "cmpunlocker VRAM unlock (64 GiB per card)", "flags": "--pipeline-parallel-size 4 --moe-backend humming --enable-prefix-caching --mamba-ssm-cache-dtype float32 --gpu-memory-utilization 0.94 --max-model-len 262144 --max-num-seqs 8 -cc.cudagraph_mode=FULL_AND_PIECEWISE --no-enable-flashinfer-autotune, MTP speculative config with num_speculative_tokens 3, --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3; env VLLM_PLE_CPU_OFFLOAD=1, VLLM_USE_FLASHINFER_SAMPLER=0, VLLM_BT_POOL=1, VLLM_PP_LAYER_PARTITION=12,12,12,12, VLLM_KV_CACHE_MEMORY_RANK0 to RANK3, NCCL_P2P_DISABLE=1, NCCL_IB_DISABLE=1; container --ipc=host --shm-size=32g", "status": "Reported", "tested_by": "flipnburner", "tested_on": "2026-09-02", "source": "https://github.com/zebgop-ops/qwen38-flashnext-pp", "files": "serve/run-pp3-mtp.sh, serve/make-patched-tree.sh, patches/ 0001 to 0016, ported-files/, tools/ (loop detector and soak harnesses), Apache-2.0", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "89", "decode_agg": "442", "agg_users": "8", "prefill_prompt": "32k tokens", "context_tokens": "262144", "p2p": "off", "pcie_link": "Gen2", "pcie_width": "x4", "power_w": "200", "spec_decode": "MTP 3", "bench_note": "PP4, MTP 3, plain decode; the 8-user total is from the patch 0016 table", "tested_on__precision": "1", "engine version": "0.1.dev20073+g8e685d198; repo commit 2f551bb (patch 0016, 2026-10-02)", "card count": "4", "decode tps": "89", "prefill tps": "11100", "vram per card": "64 GiB; --gpu-memory-utilization 0.94 plus per-rank KV budgets; measured 34.0-34.3 GiB per 12-layer rank, 37.9 GiB on the drafter rank", "power cap": "200 W", "unlock build": "cmpunlocker VRAM unlock (64 GiB per card)", "tested by": "flipnburner", "tested on": "2026-09-02", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "89", "decode agg": "442", "agg users": "8", "prefill prompt": "32k tokens", "context tokens": "262144", "pcie link": "Gen2", "pcie width": "x4", "power w": "200", "spec decode": "MTP 3", "bench note": "PP4, MTP 3, plain decode; the 8-user total is from the patch 0016 table", "tested on__precision": "1"}, {"pagetitle": "Qwen3.8 Flash Next on 4x 8 GB", "page": "Qwen3.8 Flash Next on 4x 8 GB", "model": "Qwen3.8-Flash-Next", "quant": "AWQ W4A16: wtdcode/Qwen3.8-Flash-Next-AWQ-W4A16", "engine": "wtdcode/vllm-backport (image lazymio/vllm-backport:latest-sm80)", "engine_version": "latest-sm80 tag as of 2026-10-16", "cards": "8 GB", "card_count": "4", "topology": "TP4 with expert parallel, MTP 3", "context": "1,000,000 (YaRN x4 from 262,144)", "decode_tps": "116.49", "prefill_tps": "2101.4", "vram_per_card": "64 GB", "power_cap": "", "driver": "", "unlock_build": "", "flags": "--tensor-parallel-size 4 --enable-expert-parallel --speculative-config mtp 3 --max-model-len 1000000 --max-num-seqs 8 --max-num-batched-tokens 2048 cudagraph FULL_AND_PIECEWISE --prefix-match-unit 16 --mamba-cache-mode align; VLLM_PLE_CPU_OFFLOAD=0", "status": "Reported", "tested_by": "matstatman", "tested_on": "2026-10-16", "source": "https://discord.com/channels/1518634379604136178/1528826330454429706/1560602001908170753", "files": "", "model_family": "Qwen3.8 Flash Next", "engine_family": "vLLM", "decode_single": "116.49", "decode_agg": "", "agg_users": "", "prefill_prompt": "42,584 tokens", "context_tokens": "1000000", "p2p": "", "pcie_link": "", "pcie_width": "", "power_w": "", "spec_decode": "MTP 3", "bench_note": "fresh code generation, 400 tokens, MTP 3", "tested_on__precision": "1", "engine version": "latest-sm80 tag as of 2026-10-16", "card count": "4", "decode tps": "116.49", "prefill tps": "2101.4", "vram per card": "64 GB", "power cap": "", "unlock build": "", "tested by": "matstatman", "tested on": "2026-10-16", "model family": "Qwen3.8 Flash Next", "engine family": "vLLM", "decode single": "116.49", "decode agg": "", "agg users": "", "prefill prompt": "42,584 tokens", "context tokens": "1000000", "pcie link": "", "pcie width": "", "power w": "", "spec decode": "MTP 3", "bench note": "fresh code generation, 400 tokens, MTP 3", "tested on__precision": "1"}]}