diff --git a/recipes/deepseek-v4-1-flash-1017-on-1x-cmp-170hx-xtier-engram-nvme-16k/recipe.yaml b/recipes/deepseek-v4-1-flash-1017-on-1x-cmp-170hx-xtier-engram-nvme-16k/recipe.yaml index 82e75161d..317b1bdb1 100644 --- a/recipes/deepseek-v4-1-flash-1017-on-1x-cmp-170hx-xtier-engram-nvme-16k/recipe.yaml +++ b/recipes/deepseek-v4-1-flash-1017-on-1x-cmp-170hx-xtier-engram-nvme-16k/recipe.yaml @@ -81,6 +81,55 @@ config: - '1200' - '--enable-metrics' - '--weight-loader-drop-cache-after-load' + env: + - XTIER=1 + - XTIER_THREADS=16 + - XTIER_MAIN_CPUS=auto + - XTIER_PROBE_DISPATCH_CPU=1 + - XTIER_SPIN_COUNT=1048576 + - XTIER_DYNAMIC=2 + - XTIER_DOWN_BF16=1 + - XTIER_DOWN_GRAIN=24 + - XTIER_THP=0 + - XTIER_ENGRAM_ROWS=nvme + - XTIER_ENGRAM_MODEL=$M + - XTIER_ENGRAM_CACHE_GIB=2 + - XTIER_CACHE_SLOTS=64 + - XTIER_CACHE_EVERY=4 + - XTIER_CACHE_SWAPS=16 + - XTIER_CACHE_PROMPT=1 + - XTIER_CACHE_PROMPT_KEEP=0.85 + - XTIER_CACHE_PROMPT_ROUNDS=8 + - XTIER_CACHE_PROMPT_SWAPS=512 + - XTIER_CACHE_LOG=500 + - XTIER_COLD=1 + - XTIER_COLD_PATHS=$HOME/models/ds41-cold + - XTIER_COLD_FRACTION=0.85 + - XTIER_FAST_LOAD=1 + - XTIER_PREFILL_FP4=1 + - XTIER_PREFILL_UNPACK=1 + - XTIER_PREFILL_PREFETCH=1 + - XTIER_PREFILL_OVERLAP=1 + - XTIER_PREFILL_PREFETCH_CHUNKS=6 + - XTIER_PREFILL_EXPERT_CHUNK=8 + - XTIER_PREFILL_ROUTE_ACCUM=1 + - XTIER_CPU_PREFILL_BATCH=2048 + - XTIER_GPU_PREFILL_MIN=1024 + - XTIER_HYBRID_PREFILL_MAX=1024 + - XTIER_SPLIT_PREFILL_MAX=6144 + - XTIER_DOWN_PRECISION=tf32 + - XTIER_PREFILL_TILE_M=64 + - XTIER_GATE_BN=128 + - 'XTIER_GATE_LAUNCH=8,3' + - XTIER_DOWN_BN=128 + - XTIER_DOWN_BK=64 + - 'XTIER_DOWN_LAUNCH=8,3' + - FLASHINFER_CUDA_ARCH_LIST=8.0 + - TVM_FFI_CUDA_ARCH_LIST=8.0 + - TVM_FFI_GPU_BACKEND=cuda + - HF_HUB_OFFLINE=1 + - 'PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + notes: 'Launch + env matrix transcribed verbatim from deploy/ds41.sh at recipe repo HEAD f9a8334e9f (systemd user unit deploy/ds41.service wraps it). Model dir $M=ds41-flash-1017 (491 GB bf16, 57 shards, 40 layers x 384 experts top-6, H=5120 I=2304); cold expert pool $HOME/models/ds41-cold on NVMe. 189 GB Engram table is NVMe-mmap (no VRAM/RAM residency; 2 GiB row cache). VRAM 60443/65536 MiB (92%). Cold start 150-330 s (matrix.jsonl boot_s 150; BENCH.md 330 s incl XTier pool init). Host: i9-12900 16C/24T AVX2+VNNI no AVX-512/AMX, 63 GB DDR5 dual-channel, single 2 TB 990 PRO, 63 GB swap. THREADS=16 is a hard top (17+ rejected by XTIER_NODE_CORES topology check). SLOTS=64 here means XTIER_CACHE_SLOTS (resident expert slots), NOT SGLang request slots; --max-running-requests is 2. chunked-prefill 2048 is the sweet spot: 1024 prefill-doubles the fixed overhead (26-30 tok/s), 4096 OOMs ~8500-token prompts. Spec decode is net-negative on this box (DSPARK 4.13-7.37 vs baseline 10.1; MTP/EAGLE unloadable - checkpoint is DSPARK layout, missing e_proj/h_proj/shared_head/enorm).' performance: claimed_by: submitter metrics: