{"meta":{"count":2,"page":1,"pages":1,"per_page":12,"applied":{"engine":null,"model":null,"quant":null,"storage":null,"hardware":["rtx-4090","rtx-5090"],"generation":null,"task":null,"tier":null,"tags":[],"vram":null,"search":null,"sort":"updated","featured":false},"origin":"https://localairecipes.com","openapi":"/openapi.json","updated_at":"2026-11-04T06:14:40+00:00"},"recipes":[{"slug":"qwen3-8-flash-next-iq3-s-mtp-on-1x-rtx-4090-llama-cpp-tandem-cpp","title":"Qwen3.8-Flash-Next IQ3_S + MTP on 1x RTX 4090 \u2014 llama.cpp + tandem.cpp MoE expert cache (ewhacc)","summary":"MoE expert-cache offload for llama.cpp via the tandem.cpp patch set: hot experts stay in a GPU expert cache, missed experts are split between CPU compute and an upload into the cache, sized from measured CPU and PCIe speeds; single-user decode is the target. Author's GSM8K first-200 decode numbers on the consumer PC (i5-12400F, 64 GB DDR4, RTX 4090 24 GB): Qwen3.8-Flash-Next IQ3_S + MTP 97.3 tok/s greedy and 96.1 sampled vs llama.cpp f5222f25c 29.8/29.4 (3.3x); gpt-oss-120b 51.5/48.0. REPRO CAVEAT: all numbers are author-self-reported on a repo <24h old with one commit day of history (initial commit 2026-11-02, no releases/tags); nothing reproduced on-box or by a third party - evidence is claimed, tier community. Requires CUDA 13.0-13.4 on Linux (sm_89/sm_120) and 64 GB host RAM for the 78 GiB IQ3_S weights; 1.4-2.1x gains on other MoE models that don't fit VRAM.","tier":"community","beta":false,"generation":"ada","decode_tok_s":73.7999999999999971578290569595992565155029296875,"prefill_tok_s":0,"updated_at":"2026-11-02 09:53:20.10132-07","commit_sha":"543df76dcb783c7c365b8ea88a2d81ed9d36de76","content_hash":"fd8e878ab3aa782deec1708f8b0913a0ee5ea7a7","votes":0,"credit":[{"name":"ewhacc - recipe, patch set and measurements","url":"https://github.com/ewhacc/tandem.cpp"},{"name":"llama.cpp / ggml (MIT)","url":"https://github.com/ggml-org/llama.cpp"},{"name":"MoE expert-split rule adapted from Strata (Niko1221)","url":"https://github.com/Niko1221/Strata"}],"uploaded_by":"localai-recipes","measured":null,"links":{"self":"/api/v1/recipes/qwen3-8-flash-next-iq3-s-mtp-on-1x-rtx-4090-llama-cpp-tandem-cpp.json","alternate":"/r/qwen3-8-flash-next-iq3-s-mtp-on-1x-rtx-4090-llama-cpp-tandem-cpp"}},{"slug":"deepseek-v4-1-flash-1017-streaming-nvme-stack-on-1x-rtx-5090-fro","title":"DeepSeek-V4.1-Flash-1017 streaming NVMe stack on 1x RTX 5090 (fronteagle739)","summary":"Streams a 483 GiB DeepSeek-V4.1-Flash-1017 pack from NVMe straight into one 32 GB RTX 5090, so a frontier-size MoE runs on a desktop with 32 GB of VRAM and 32 GB of system RAM. Windows-only: the stack is built on DirectStorage (a Windows API) and ships as PowerShell launchers plus a Windows ds4-server.exe, so prefill cannot run on Linux - see install_note. Packed-FP4 measures 200K real-text prefill at 43.2 s (4,632 tok/s) with 7.6-7.9 tok/s max-quality decode and 22.7 tok/s max-speed; Q4_K/Q8_0 measures 3,356 tok/s prefill and 7.5 / 17.8 tok/s decode. An optional 512 GB M3 Ultra can decode while the RTX prefills. Weights are never requantized on load; a quality/speed dial trades exactness for cached-expert reuse.","tier":"community","beta":false,"generation":"blackwell","decode_tok_s":12.699999999999999289457264239899814128875732421875,"prefill_tok_s":3994,"updated_at":"2026-10-31 12:32:41.796895-06","commit_sha":"1926a9bbc1be021dbef26cb9199faf4531de2b9f","content_hash":"f40ac13b6d6e0cf49620410496b08fb7ce1ae6f6","votes":0,"credit":[{"name":"fronteagle739 - recipe, streaming stack and measurements","url":"https://github.com/fronteagle739/streaming-llama-stack"},{"name":"llama.cpp / ggml (MIT)","url":"https://github.com/ggml-org/llama.cpp"},{"name":"antirez/ds4 (MIT)","url":"https://github.com/antirez/ds4"},{"name":"IngeniousIdiocy/ds4-v41-m3ultra (MIT)","url":"https://github.com/IngeniousIdiocy/ds4-v41-m3ultra"},{"name":"DeepSeek - DeepSeek-V4.1-Flash-1017 base model","url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017"}],"uploaded_by":"localai-recipes","measured":null,"links":{"self":"/api/v1/recipes/deepseek-v4-1-flash-1017-streaming-nvme-stack-on-1x-rtx-5090-fro.json","alternate":"/r/deepseek-v4-1-flash-1017-streaming-nvme-stack-on-1x-rtx-5090-fro"}}]}