diff --git a/recipes/qwen3-8-flash-next-nvfp4-dgx-spark-gb10-sglang-nextn-mtp/recipe.yaml b/recipes/qwen3-8-flash-next-nvfp4-dgx-spark-gb10-sglang-nextn-mtp/recipe.yaml deleted file mode 100644 index 5237e62a0..000000000 --- a/recipes/qwen3-8-flash-next-nvfp4-dgx-spark-gb10-sglang-nextn-mtp/recipe.yaml +++ /dev/null @@ -1,140 +0,0 @@ -slug: qwen3-8-flash-next-nvfp4-dgx-spark-gb10-sglang-nextn-mtp -title: 'Qwen3.8-Flash-Next NVFP4 DGX Spark GB10 SGLang NEXTN MTP' -engine: - name: sglang - version: 'image tag qwen38-flash:v1.5 (FROM lmsysorg/sglang:latest)' - source_type: docker_image - git_url: 'https://github.com/bemlerlabs/qwen3.8-flash-next-dgx-spark-sglang' - pinned_commit: 06fd72a673284696e55b52b3adb58aa3cc6daf35 - image_digest: 'qwen38-flash:v1.5' - install_note: 'docker build from the repo Dockerfile @ 06fd72a6: FROM lmsysorg/sglang:latest (UNPINNED upstream base - rebuilds may drift; env TORCH_CUDA_ARCH_LIST=12.0;12.1, ENTRYPOINT python3 -m sglang.launch_server), tagged qwen38-flash:v1.5 locally. launch.sh runs it with --gpus all --memory 110g --shm-size 16g --network host and the NVMe/JIT-cache mounts. No digest pin on the base image - author ships tag-level reproducibility only (honest gap vs the Felliks/YSLAB pin discipline).' -summary: 'Production single-node DGX Spark (Grace Blackwell GB10, SM121, 128GB unified memory) recipe for Qwen3.8-Flash-Next on SGLang: ModelOpt NVFP4 quantization with zero-copy NVMe mmap of the 47.7 GiB ple_table.bin via --ple-offload-embedding (eliminates 51 GiB of unified-memory footprint), custom Triton FP4 prefill kernels + trtllm_mha decode kernels on SM121, NEXTN MTP (3 verification steps / 4 draft tokens) lifting output to 110-152 tok/s, GDN+QSA hybrid (131,072-token served window, 262k native capacity), 96-slot mamba recurrence buffers, sharp froggeric v22.1 Jinja chat template (-18% response length), persistent Triton/Inductor SM121 JIT caches on NVMe, 8-worker parallel NVMe pre-warm (206 shards in 17s) + 16-core Grace OMP. Author-measured vs same-box 27B dense baseline: 110.4-152.8 tok/s decode w/ MTP (baseline 12.1 base / 21.5 MTP), single-stream CoT 25.1-28.8 tok/s (2.62x speculative gain), TTFT ~347ms radix hit / ~465ms cold, 113 GiB / 121 GiB RAM used (~8 GiB headroom), 103.5 GiB' -model: - name: Qwen3.8-Flash-Next-NVFP4 - family: qwen3-8-flash-next - hf_repo: RadixArk/Qwen3.8-Flash-Next-NVFP4 - hf_revision: 7b719225242aacd3dbd3f9407468c2ee9a9d2594 - quant: nvfp4 -hardware: - - - vendor: nvidia - card: 'DGX Spark (GB10, unified memory)' - vram_gb: 128 - device_count: 1 -tasks: - - chat - - code -tags: - - bemlerlabs - - sglang - - dgx-spark - - gb10 - - sm121 - - nvme-ple - - mtp-speculative - - triton-prefill - - trtllm-mha -config: - launch: - - python3 - - '-m' - - sglang.launch_server - - '--model-path' - - RadixArk/Qwen3.8-Flash-Next-NVFP4 - - '--revision' - - 7b719225242aacd3dbd3f9407468c2ee9a9d2594 - - '--tp-size' - - '1' - - '--served-model-name' - - qwen3.8-flash-next - - '--quantization' - - modelopt_fp4 - - '--prefill-attention-backend' - - triton - - '--decode-attention-backend' - - trtllm_mha - - '--ple-offload-embedding' - - '--mamba-radix-cache-strategy' - - extra_buffer - - '--max-mamba-cache-size' - - '96' - - '--mem-fraction-static' - - '0.90' - - '--context-length' - - '131072' - - '--chunked-prefill-size' - - '4096' - - '--enable-mixed-chunk' - - '--schedule-conservativeness' - - '0.8' - - '--max-running-requests' - - '4' - - '--allow-auto-truncate' - - '--speculative-algorithm' - - NEXTN - - '--speculative-num-steps' - - '3' - - '--speculative-eagle-topk' - - '1' - - '--speculative-num-draft-tokens' - - '4' - - '--speculative-draft-model-quantization' - - unquant - - '--reasoning-parser' - - qwen3 - - '--tool-call-parser' - - qwen3_coder - - '--chat-template' - - /out/chat-template-flashnext.jinja - - '--api-key' - - local - - '--host' - - 0.0.0.0 - - '--port' - - '8000' - runtime: - env: 'SGLANG_QWEN4_PLE_MMAP_DIR=/ple (47.7 GiB ple_table.bin, zero-copy mmap) | TRITON_CACHE_DIR=/root/.cache/sglang/triton | TORCHINDUCTOR_CACHE_DIR=/root/.cache/sglang/inductor | OMP_NUM_THREADS=16 | MKL_NUM_THREADS=16' - notes: "Serve profile = launch.sh @ 06fd72a6 verbatim (docker run --rm --name qwen38-flash --gpus all --memory 110g --memory-swap 110g --shm-size 16g --network host --ipc=host, mounts: HF cache, /out config dir w/ chat-template-flashnext.jinja, /ple PLE dir, sglang cache). Model downloaded at pinned revision (hf snapshot under ~/.cache/huggingface); 8-worker parallel pre-warm cats 206 safetensors shards + 4-worker pre-warm of the PLE table into page cache (<25s) before docker exec. Background health monitor polls :8000/health every 10s x270. Rollback note per README: the 0.90 mem-fraction + full-PLE state is the HEAD 'perfectly working state' (commit subject); systemd unit systemd/qwen38-flash.service provided for 24/7 with crash recovery. Sampling: OpenAI-compatible :8000/v1, api_key 'local' default. Chat template: repo file chat-template-flashnext.jinja (sharp froggeric v22.1, mounted READ-ONLY via /out)." -performance: - claimed_by: submitter - metrics: - tok_s: 110.4 - context_len: 131072 - notes: 'Author-measured (README telemetry table @ 06fd72a6 + Qwen3.8_FlashNext_DGX_Spark_Benchmark_Report.pdf in-repo, single DGX Spark GB10 @ 273 GB/s): decode 110.4-152.8 tok/s with NEXTN MTP; single-stream CoT 25.1-28.8 tok/s (2.62x speculative gain); TTFT ~347ms radix hit / ~465ms cold; total RAM 113 GiB / 121 GiB (~8 GiB headroom); SGLang VRAM allocation 103.5 GiB static 0.90; PLE table 47.7 GiB zero-copy mmap; swap buffer 1.2 GiB / 99 GiB free; cold boot/restart ~8.5 min (51.8s prefetch, zero swap' - points: - - - context_len: 131072 - concurrency: 1 - decode_tok_s: 110.4 - ttft_ms: 465 - notes: 'README telemetry @ 06fd72a6: MTP decode band 110.4-152.8 tok/s; TTFT ~347ms radix hit / ~465ms cold; NEXTN 3 steps / 4 draft' - - - context_len: 131072 - concurrency: 1 - decode_tok_s: 152.8 - notes: 'README telemetry: upper end of the measured 110.4-152.8 tok/s MTP band' -artifacts: - - - kind: model_weights - pointer_type: hf_model - pointer: 'https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4' - pin: "revision 7b719225242aacd3dbd3f9407468c2ee9a9d2594 (MODEL_REVISION default in launch.sh) - 126GB NVFP4 checkpoint shards + 51GB ple_table.bin mmap'd from NVMe" - - - kind: engine_image - pointer_type: docker_digest - pointer: 'https://hub.docker.com/r/lmsysorg/sglang' - pin: 'FROM lmsysorg/sglang:latest built per repo Dockerfile @ 06fd72a6 -> local tag qwen38-flash:v1.5 (base tag UNPINNED upstream - noted in install_note); TORCH_CUDA_ARCH_LIST=12.0;12.1' - - - kind: reference - pointer_type: git_commit - pointer: 'https://github.com/bemlerlabs/qwen3.8-flash-next-dgx-spark-sglang' - pin: '06fd72a673284696e55b52b3adb58aa3cc6daf35 - launch.sh (full argv), Dockerfile, chat-template-flashnext.jinja, systemd/qwen38-flash.service, benchmark.py + PDF benchmark report' -provenance: - source_urls: - - 'https://github.com/bemlerlabs/qwen3.8-flash-next-dgx-spark-sglang' - - 'https://github.com/bemlerlabs/qwen3.8-flash-next-dgx-spark-sglang/blob/main/launch.sh' - credit: - - - name: 'Michael Bemler (Bemler Labs)' - url: 'https://github.com/bemlerlabs/qwen3.8-flash-next-dgx-spark-sglang' - notes: 'Ingested from GitHub repo bemlerlabs/qwen3.8-flash-next-dgx-spark-sglang @ 06fd72a673284696e55b52b3adb58aa3cc6daf35 (wave 3, kanban t_21a3a083). Recipe architected by Michael Bemler per README. First DGX Spark + first SGLang-serving recipe on the site (new hardware family dgx-spark seeded via 0051). Apache-2.0.'