{"meta":{"count":2,"page":1,"pages":1,"per_page":12,"applied":{"engine":null,"model":null,"quant":null,"storage":null,"hardware":["rtx-5090-x2","rtx-5090"],"generation":null,"task":null,"tier":null,"tags":[],"vram":null,"search":null,"sort":"updated","featured":false},"origin":"https://localairecipes.com","openapi":"/openapi.json","updated_at":"2026-11-01T09:25:59+00:00"},"recipes":[{"slug":"deepseek-v4-1-flash-1017-streaming-nvme-stack-on-1x-rtx-5090-fro","title":"DeepSeek-V4.1-Flash-1017 streaming NVMe stack on 1x RTX 5090 (fronteagle739)","summary":"Streams a 483 GiB DeepSeek-V4.1-Flash-1017 pack from NVMe straight into one 32 GB RTX 5090, so a frontier-size MoE runs on a desktop with 32 GB of VRAM and 32 GB of system RAM. Windows-only: the stack is built on DirectStorage (a Windows API) and ships as PowerShell launchers plus a Windows ds4-server.exe, so prefill cannot run on Linux - see install_note. Packed-FP4 measures 200K real-text prefill at 43.2 s (4,632 tok/s) with 7.6-7.9 tok/s max-quality decode and 22.7 tok/s max-speed; Q4_K/Q8_0 measures 3,356 tok/s prefill and 7.5 / 17.8 tok/s decode. An optional 512 GB M3 Ultra can decode while the RTX prefills. Weights are never requantized on load; a quality/speed dial trades exactness for cached-expert reuse.","tier":"community","beta":false,"generation":"blackwell","decode_tok_s":12.699999999999999289457264239899814128875732421875,"prefill_tok_s":3994,"updated_at":"2026-10-31 12:32:41.796895-06","commit_sha":"1926a9bbc1be021dbef26cb9199faf4531de2b9f","content_hash":"f40ac13b6d6e0cf49620410496b08fb7ce1ae6f6","votes":0,"credit":[{"name":"fronteagle739 - recipe, streaming stack and measurements","url":"https://github.com/fronteagle739/streaming-llama-stack"},{"name":"llama.cpp / ggml (MIT)","url":"https://github.com/ggml-org/llama.cpp"},{"name":"antirez/ds4 (MIT)","url":"https://github.com/antirez/ds4"},{"name":"IngeniousIdiocy/ds4-v41-m3ultra (MIT)","url":"https://github.com/IngeniousIdiocy/ds4-v41-m3ultra"},{"name":"DeepSeek - DeepSeek-V4.1-Flash-1017 base model","url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017"}],"uploaded_by":"localai-recipes","measured":null,"links":{"self":"/api/v1/recipes/deepseek-v4-1-flash-1017-streaming-nvme-stack-on-1x-rtx-5090-fro.json","alternate":"/r/deepseek-v4-1-flash-1017-streaming-nvme-stack-on-1x-rtx-5090-fro"}},{"slug":"glm-5-3-flash-exl3-2-05bpw-on-2x-rtx-5090-tabbyapi-hot-expert-sw","title":"GLM-5.3-Flash EXL3 2.05bpw on 2x RTX 5090 - TabbyAPI hot-expert swaps, 262K ctx, vision","summary":"GLM-5.3-Flash (321B MoE, ~18B active) turboderp EXL3 2.05bpw on two RTX 5090 + 64 GB DDR5: TabbyAPI on ExLlamaV3 with CPU-offloaded routed experts (96/layer GPU0 layers 3-22+MTP, 102/layer GPU1 23-44), runtime hot-expert swaps, per-GPU expert split, the DSA indexer's per-token rows in a fixed ring, live /metrics token counters, the expert-exchange clock handed to the first text MoE layer so swaps also run at 2-4 streams, MTP head drafting 1 tok/step single-stream, 262,144-token window on an 8-bit KV page pool (~1.6 GiB across 4), vision ON with tower linears pinned in host RAM. Exact-output: no expert skipped, every draft verified. Measured 2026-10-10 (R969, three fresh boots): c1 score 62.9 tok/s (code 59.9/prose 66.7/chat 68.5/html 64.8/edit 53.9), 2/3/4-stream sums 74.4/92.0/98.0. Nine in-repo overlays on the qwen3.8-flash-next-2x-rtx5090 base image.","tier":"community","beta":false,"generation":"blackwell","decode_tok_s":62.89999999999999857891452847979962825775146484375,"prefill_tok_s":2023,"updated_at":"2026-10-29 23:29:37.491895-06","commit_sha":"4dc491797af361a6a0f4cdedde5f7dbeb918082d","content_hash":"66ee933dc865f3362d1e2880b2ef8e330f331b05","votes":0,"credit":[{"name":"adrienbrault (recipe, five patch overlays, measurements)","url":"https://github.com/adrienbrault/glm5.3-flash-2x-rtx5090"},{"name":"zai-org (GLM-5.3-Flash)","url":"https://huggingface.co/zai-org/GLM-5.3-Flash"},{"name":"turboderp (EXL3 2.05bpw quantisation; ExLlamaV3 + CPU MoE offload)","url":"https://huggingface.co/turboderp/GLM-5.3-Flash-exl3"},{"name":"theroyallab (TabbyAPI)","url":"https://github.com/theroyallab/tabbyAPI"},{"name":"adrienbrault/qwen3.8-flash-next-2x-rtx5090 (base image: ExLlamaV3 dev 5783a93 patch stack + TabbyAPI 53da7919)","url":"https://github.com/adrienbrault/qwen3.8-flash-next-2x-rtx5090"}],"uploaded_by":"localai-recipes","measured":null,"links":{"self":"/api/v1/recipes/glm-5-3-flash-exl3-2-05bpw-on-2x-rtx-5090-tabbyapi-hot-expert-sw.json","alternate":"/r/glm-5-3-flash-exl3-2-05bpw-on-2x-rtx-5090-tabbyapi-hot-expert-sw"}}]}