LOCALAIRECIPES / BENCHMARK RUN REPORT
spot-check llm-inference-bench 0.8.0 exllamav3 0f685aa + row32 four-slot profile layout: tp ×TP2 on 2x CMP170HX 64GiB, 4 active sequences GLM-5.3-Flash
attached to /r/glm-5-3-flash-high-effort-on-2x-cmp170hx-exl3-tp2-four-slots variant: GLM-5.3-Flash
Uploaded report
report.md (markdown, shown as-is)
# GLM-5.3-Flash, high effort, on 2x CMP170HX Four native slots; EXL3 3.05bpw with the Weightless GLP44 steering vector at alpha 2, GPU-resident Q8 KV. 1, 2 and 4 agents at short and nominal 32K/128K context, one 30-second run per cell. Exact completion-token usage, including reasoning; actual input lengths are retained. A separate four-request 512-token completion cohort. Inference speed only, not accuracy or intelligence. Differences between efforts are not separated from run-to-run variation. Full methods, tables and caveats: https://github.com/alexvnesta/glm53-flash-cmp170hx-tp/blob/fa0adf1899015f692ddce30391ee7e740a0910c9/docs/COMMUNITY_BENCHMARK_2026-10-17.md
Run file
raw JSON (verbatim; gpu uuids + private hosts masked)
{
"tier": "spot-check",
"tool": "llm-inference-bench",
"model": "GLM-5.3-Flash",
"driver": "610.57.04",
"engine": "exllamav3",
"results": [
{
"ttft_avg": 0.31440564501099288463592529296875,
"ttft_p50": 0.31440564501099288463592529296875,
"ttft_p90": 0.31440564501099288463592529296875,
"ttft_p99": 0.31440564501099288463592529296875,
"wall_time": 35.5285498080193065106868743896484375,
"num_errors": 0,
"concurrency": 1,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=1/1, stable=3.0s",
"total_tokens": 1293,
"aggregate_tps": 43.16218167036803521341425948776304721832275390625,
"loop_detected": false,
"num_completed": 1,
"request_count": 1,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 0,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.31440564501099288463592529296875,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 1548,
"output_tps_per_user": 44.00383593341715737778940820135176181793212890625,
"time_to_second_token": 0.0565635859966278076171875,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.0227252915294274444868438678213351522572338581085205078125,
"chunk_inter_token_latency_avg": 0.0227399909418009409101113504902968998067080974578857421875
}
],
"warmup_duration": 5.51400000000000023447910280083306133747100830078125,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 54.3900000000000005684341886080801486968994140625,
"temp_max_c": 56,
"vram_total_mb": 131072,
"cpu_temp_max_c": 69.3799999999999954525264911353588104248046875,
"cpu_util_avg_pct": 10.8599999999999994315658113919198513031005859375,
"duration_seconds": 28.73400000000000176214598468504846096038818359375,
"gpu_util_avg_pct": 87.2900000000000062527760746888816356658935546875,
"gpu_util_max_pct": 93,
"mem_util_avg_pct": 14.21000000000000085265128291212022304534912109375,
"mem_util_max_pct": 15,
"pcie_rx_avg_mb_s": 499.3600000000000136424205265939235687255859375,
"pcie_rx_max_mb_s": 643,
"pcie_tx_avg_mb_s": 462.6399999999999863575794734060764312744140625,
"pcie_tx_max_mb_s": 524,
"vram_used_avg_mb": 126720,
"vram_used_max_mb": 126720,
"power_total_avg_w": 338.779999999999972715158946812152862548828125,
"power_total_max_w": 369.06999999999999317878973670303821563720703125,
"vram_used_avg_pct": 96.68000000000000682121026329696178436279296875,
"vram_used_max_pct": 96.68000000000000682121026329696178436279296875,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 78,
"output_seq_len_avg": 1548,
"output_seq_len_p50": 1548,
"output_seq_len_p90": 1548,
"output_seq_len_p99": 1548,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.956779999999998409521140274591743946075439453125,
"per_request_avg_tps": 43.16218167036803521341425948776304721832275390625,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1293,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.0227252915294274444868438678213351522572338581085205078125,
"inter_token_latency_p50": 0.0227252915294274444868438678213351522572338581085205078125,
"inter_token_latency_p90": 0.0227252915294274444868438678213351522572338581085205078125,
"inter_token_latency_p99": 0.0227252915294274444868438678213351522572338581085205078125,
"output_tps_per_user_avg": 44.00383593341715737778940820135176181793212890625,
"output_tps_per_user_p50": 44.00383593341715737778940820135176181793212890625,
"output_tps_per_user_p90": 44.00383593341715737778940820135176181793212890625,
"output_tps_per_user_p99": 44.00383593341715737778940820135176181793212890625,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.00102199999999896817826083861291408538818359375,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.0565635859966278076171875,
"time_to_second_token_p50": 0.0565635859966278076171875,
"time_to_second_token_p90": 0.0565635859966278076171875,
"time_to_second_token_p99": 0.0565635859966278076171875,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.0227399909418009409101113504902968998067080974578857421875,
"chunk_inter_token_latency_p50": 0.0227399909418009409101113504902968998067080974578857421875,
"chunk_inter_token_latency_p90": 0.0227399909418009409101113504902968998067080974578857421875,
"chunk_inter_token_latency_p99": 0.0227399909418009409101113504902968998067080974578857421875
},
{
"ttft_avg": 0.57007440450252033770084381103515625,
"ttft_p50": 0.57007440450252033770084381103515625,
"ttft_p90": 0.5705039921042043271626198475132696330547332763671875,
"ttft_p99": 0.57060064931458331916047654885915108025074005126953125,
"wall_time": 35.5177052859799005091190338134765625,
"num_errors": 0,
"concurrency": 2,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=2/2, stable=3.0s",
"total_tokens": 1791,
"aggregate_tps": 59.70111690832836615072665154002606868743896484375,
"loop_detected": false,
"num_completed": 2,
"request_count": 2,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 0,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.5695374200004152953624725341796875,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 1068,
"output_tps_per_user": 30.534436356713026583520331769250333309173583984375,
"time_to_second_token": 0.077509408001787960529327392578125,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.03274990860540803738754078722195117734372615814208984375,
"chunk_inter_token_latency_avg": 0.032780630846126056210376731314681819640100002288818359375
},
{
"ttft": 0.570611389004625380039215087890625,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 1067,
"output_tps_per_user": 30.579156875940956439308138214983046054840087890625,
"time_to_second_token": 0.0776753919781185686588287353515625,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.032702013468094641435168767884533735923469066619873046875,
"chunk_inter_token_latency_avg": 0.032732719584027118731039962540307897143065929412841796875
}
],
"warmup_duration": 5.51400000000000023447910280083306133747100830078125,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 53.67999999999999971578290569595992565155029296875,
"temp_max_c": 55,
"vram_total_mb": 131072,
"cpu_temp_max_c": 69.8799999999999954525264911353588104248046875,
"cpu_util_avg_pct": 24.760000000000001563194018672220408916473388671875,
"duration_seconds": 28.803000000000000824229573481716215610504150390625,
"gpu_util_avg_pct": 80.5,
"gpu_util_max_pct": 88,
"mem_util_avg_pct": 10.6099999999999994315658113919198513031005859375,
"mem_util_max_pct": 12,
"pcie_rx_avg_mb_s": 403.56999999999999317878973670303821563720703125,
"pcie_rx_max_mb_s": 723,
"pcie_tx_avg_mb_s": 376.8600000000000136424205265939235687255859375,
"pcie_tx_max_mb_s": 435,
"vram_used_avg_mb": 126720,
"vram_used_max_mb": 126720,
"power_total_avg_w": 304.970000000000027284841053187847137451171875,
"power_total_max_w": 365.70999999999997953636921010911464691162109375,
"vram_used_avg_pct": 96.68000000000000682121026329696178436279296875,
"vram_used_max_pct": 96.68000000000000682121026329696178436279296875,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 78,
"output_seq_len_avg": 1067.5,
"output_seq_len_p50": 1067.5,
"output_seq_len_p90": 1067.90000000000009094947017729282379150390625,
"output_seq_len_p99": 1067.990000000000009094947017729282379150390625,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.999438999999998856083038845099508762359619140625,
"per_request_avg_tps": 29.850558454164183075363325770013034343719482421875,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1791,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.03272596103675133594190782559962826780974864959716796875,
"inter_token_latency_p50": 0.03272596103675133594190782559962826780974864959716796875,
"inter_token_latency_p90": 0.03274511909167669709841419489748659543693065643310546875,
"inter_token_latency_p99": 0.0327494296540349061341856895523960702121257781982421875,
"output_tps_per_user_avg": 30.5567966163269915114142349921166896820068359375,
"output_tps_per_user_p50": 30.5567966163269915114142349921166896820068359375,
"output_tps_per_user_p90": 30.5746848240181634537293575704097747802734375,
"output_tps_per_user_p99": 30.578709670748676074936156510375440120697021484375,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.000592999999998511384546873159706592559814453125,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.07759239998995326459407806396484375,
"time_to_second_token_p50": 0.07759239998995326459407806396484375,
"time_to_second_token_p90": 0.07765879358048551062143616263711010105907917022705078125,
"time_to_second_token_p99": 0.07767373213835526701842582042445428669452667236328125,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.032756675215076587470708346927494858391582965850830078125,
"chunk_inter_token_latency_p50": 0.032756675215076587470708346927494858391582965850830078125,
"chunk_inter_token_latency_p90": 0.032775839719916162462443054437244427390396595001220703125,
"chunk_inter_token_latency_p99": 0.032780151733505069611140925189829431474208831787109375
},
{
"ttft_avg": 1.088150133480667136609554290771484375,
"ttft_p50": 1.08851832500658929347991943359375,
"ttft_p90": 1.089313965663313954479463063762523233890533447265625,
"ttft_p99": 1.089588523524580576662401654175482690334320068359375,
"wall_time": 35.5980308209545910358428955078125,
"num_errors": 0,
"concurrency": 4,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=4/4, stable=3.0s",
"total_tokens": 2884,
"aggregate_tps": 96.2687441498165981101919896900653839111328125,
"loop_detected": false,
"num_completed": 4,
"request_count": 4,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 0,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 1.0896190299536101520061492919921875,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 844,
"output_tps_per_user": 24.51998924293135218022143817506730556488037109375,
"time_to_second_token": 0.1336989430128596723079681396484375,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.040783052149514338602376284370620851404964923858642578125,
"chunk_inter_token_latency_avg": 0.04083148807843299532560621400989475660026073455810546875
},
{
"ttft": 1.08594485395587980747222900390625,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 844,
"output_tps_per_user": 24.517640818675406677584760473109781742095947265625,
"time_to_second_token": 0.133106947992928326129913330078125,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.040786958557541434788884515683093923144042491912841796875,
"chunk_inter_token_latency_avg": 0.040835399125899562999819636388565413653850555419921875
},
{
"ttft": 1.0886021489859558641910552978515625,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 844,
"output_tps_per_user": 24.51969023207806230857386253774166107177734375,
"time_to_second_token": 0.1336179290083236992359161376953125,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.0407835494875764259514738796497113071382045745849609375,
"chunk_inter_token_latency_avg": 0.04083198600715787218717878204188309609889984130859375
},
{
"ttft": 1.0884345010272227227687835693359375,
"latency": 0,
"completed": false,
"input_tokens": 78,
"output_tokens": 844,
"output_tps_per_user": 24.51972880338650639941988629288971424102783203125,
"time_to_second_token": 0.1336431169766001403331756591796875,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.04078348533210067461407533073725062422454357147216796875,
"chunk_inter_token_latency_avg": 0.040831921775487965697859493729993118904531002044677734375
}
],
"warmup_duration": 5.51199999999999956656893118633888661861419677734375,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 54.17999999999999971578290569595992565155029296875,
"temp_max_c": 56,
"vram_total_mb": 131072,
"cpu_temp_max_c": 68.8799999999999954525264911353588104248046875,
"cpu_util_avg_pct": 20.969999999999998863131622783839702606201171875,
"duration_seconds": 28.721000000000000085265128291212022304534912109375,
"gpu_util_avg_pct": 79.3599999999999994315658113919198513031005859375,
"gpu_util_max_pct": 88,
"mem_util_avg_pct": 8.78999999999999914734871708787977695465087890625,
"mem_util_max_pct": 10,
"pcie_rx_avg_mb_s": 550.1399999999999863575794734060764312744140625,
"pcie_rx_max_mb_s": 876,
"pcie_tx_avg_mb_s": 465.56999999999999317878973670303821563720703125,
"pcie_tx_max_mb_s": 784,
"vram_used_avg_mb": 126720,
"vram_used_max_mb": 126720,
"power_total_avg_w": 313.1399999999999863575794734060764312744140625,
"power_total_max_w": 368.490000000000009094947017729282379150390625,
"vram_used_avg_pct": 96.68000000000000682121026329696178436279296875,
"vram_used_max_pct": 96.68000000000000682121026329696178436279296875,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 78,
"output_seq_len_avg": 844,
"output_seq_len_p50": 844,
"output_seq_len_p90": 844,
"output_seq_len_p99": 844,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.9578020000000009304130799137055873870849609375,
"per_request_avg_tps": 24.067186037454149527547997422516345977783203125,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 2884,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.04078426138168321501975555065655498765408992767333984375,
"inter_token_latency_p50": 0.040783517409838550282774605193480965681374073028564453125,
"inter_token_latency_p90": 0.040785935836551934219329496045247651636600494384765625,
"inter_token_latency_p99": 0.040786856285442485425818404110032133758068084716796875,
"output_tps_per_user_avg": 24.5192622742678310032715671695768833160400390625,
"output_tps_per_user_p50": 24.519709517732284353996874415315687656402587890625,
"output_tps_per_user_p90": 24.519911111067898445980972610414028167724609375,
"output_tps_per_user_p99": 24.519981429745005385711920098401606082916259765625,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.001000000000001222133505507372319698333740234375,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.133516734247677959501743316650390625,
"time_to_second_token_p50": 0.1336305229924619197845458984375,
"time_to_second_token_p90": 0.133682195201981823817760641759377904236316680908203125,
"time_to_second_token_p99": 0.1336972682317718874589473898595315404236316680908203125,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.04083269874674459731789255556577700190246105194091796875,
"chunk_inter_token_latency_p50": 0.04083195389132292241196608983955229632556438446044921875,
"chunk_inter_token_latency_p90": 0.040834375190277057143806160866006393916904926300048828125,
"chunk_inter_token_latency_p99": 0.04083529673233730894477133688269532285630702972412109375
},
{
"ttft_avg": 0.4427903559990227222442626953125,
"ttft_p50": 0.4427903559990227222442626953125,
"ttft_p90": 0.4427903559990227222442626953125,
"ttft_p99": 0.4427903559990227222442626953125,
"wall_time": 56.5867666230187751352787017822265625,
"num_errors": 0,
"concurrency": 1,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=1/1, stable=3.0s",
"total_tokens": 1279,
"aggregate_tps": 42.67706541990319379920038045383989810943603515625,
"loop_detected": false,
"num_completed": 1,
"request_count": 1,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 32768,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.4427903559990227222442626953125,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 1455,
"output_tps_per_user": 44.0162495028323945689407992176711559295654296875,
"time_to_second_token": 0.0606395289651118218898773193359375,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.0227188824876061075397348787419105065055191516876220703125,
"chunk_inter_token_latency_avg": 0.022876215468822215515221074610963114537298679351806640625
}
],
"warmup_duration": 26.556000000000000937916411203332245349884033203125,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 55.78999999999999914734871708787977695465087890625,
"temp_max_c": 57,
"vram_total_mb": 131072,
"cpu_temp_max_c": 66.5,
"cpu_util_avg_pct": 9.6899999999999995026200849679298698902130126953125,
"duration_seconds": 28.70700000000000073896444519050419330596923828125,
"gpu_util_avg_pct": 88.43000000000000682121026329696178436279296875,
"gpu_util_max_pct": 93,
"mem_util_avg_pct": 14.1099999999999994315658113919198513031005859375,
"mem_util_max_pct": 15,
"pcie_rx_avg_mb_s": 521.1399999999999863575794734060764312744140625,
"pcie_rx_max_mb_s": 654,
"pcie_tx_avg_mb_s": 436.5,
"pcie_tx_max_mb_s": 510,
"vram_used_avg_mb": 128040,
"vram_used_max_mb": 128040,
"power_total_avg_w": 349.33999999999997498889570124447345733642578125,
"power_total_max_w": 374.8999999999999772626324556767940521240234375,
"vram_used_avg_pct": 97.68999999999999772626324556767940521240234375,
"vram_used_max_pct": 97.68999999999999772626324556767940521240234375,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 32314,
"output_seq_len_avg": 1455,
"output_seq_len_p50": 1455,
"output_seq_len_p90": 1455,
"output_seq_len_p99": 1455,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.969257999999999952933649183250963687896728515625,
"per_request_avg_tps": 42.67706541990319379920038045383989810943603515625,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1279,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.0227188824876061075397348787419105065055191516876220703125,
"inter_token_latency_p50": 0.0227188824876061075397348787419105065055191516876220703125,
"inter_token_latency_p90": 0.0227188824876061075397348787419105065055191516876220703125,
"inter_token_latency_p99": 0.0227188824876061075397348787419105065055191516876220703125,
"output_tps_per_user_avg": 44.0162495028323945689407992176711559295654296875,
"output_tps_per_user_p50": 44.0162495028323945689407992176711559295654296875,
"output_tps_per_user_p90": 44.0162495028323945689407992176711559295654296875,
"output_tps_per_user_p99": 44.0162495028323945689407992176711559295654296875,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.000416999999998779458110220730304718017578125,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.0606395289651118218898773193359375,
"time_to_second_token_p50": 0.0606395289651118218898773193359375,
"time_to_second_token_p90": 0.0606395289651118218898773193359375,
"time_to_second_token_p99": 0.0606395289651118218898773193359375,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.022876215468822215515221074610963114537298679351806640625,
"chunk_inter_token_latency_p50": 0.022876215468822215515221074610963114537298679351806640625,
"chunk_inter_token_latency_p90": 0.022876215468822215515221074610963114537298679351806640625,
"chunk_inter_token_latency_p99": 0.022876215468822215515221074610963114537298679351806640625
},
{
"ttft_avg": 0.71872195802279748022556304931640625,
"ttft_p50": 0.71872195802279748022556304931640625,
"ttft_p90": 0.71916384522919540867036403142265044152736663818359375,
"ttft_p99": 0.71926326985063493424377156770788133144378662109375,
"wall_time": 35.1167293630423955619335174560546875,
"num_errors": 0,
"concurrency": 2,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=2/2, stable=3.0s",
"total_tokens": 1320,
"aggregate_tps": 44.0671361732829183210924384184181690216064453125,
"loop_detected": false,
"num_completed": 2,
"request_count": 2,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 32768,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.719274317030794918537139892578125,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 743,
"output_tps_per_user": 21.985131013819387391095006023533642292022705078125,
"time_to_second_token": 0.124890203005634248256683349609375,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.045485287277634199598619346716077416203916072845458984375,
"chunk_inter_token_latency_avg": 0.04554667093118026921505503423759364522993564605712890625
},
{
"ttft": 0.7181695990148000419139862060546875,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 743,
"output_tps_per_user": 21.98415342814559636508420226164162158966064453125,
"time_to_second_token": 0.1248798330198042094707489013671875,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.0454873099056765373138233599092927761375904083251953125,
"chunk_inter_token_latency_avg": 0.045548696288815103405678286208058125339448451995849609375
}
],
"warmup_duration": 5.02400000000000002131628207280300557613372802734375,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 13,
"gpu_count": 2,
"temp_avg_c": 52.5,
"temp_max_c": 54,
"vram_total_mb": 131072,
"cpu_temp_max_c": 69.5,
"cpu_util_avg_pct": 19.760000000000001563194018672220408916473388671875,
"duration_seconds": 26.562999999999998834709913353435695171356201171875,
"gpu_util_avg_pct": 82.1200000000000045474735088646411895751953125,
"gpu_util_max_pct": 91,
"mem_util_avg_pct": 8.3800000000000007815970093361102044582366943359375,
"mem_util_max_pct": 9,
"pcie_rx_avg_mb_s": 303.6200000000000045474735088646411895751953125,
"pcie_rx_max_mb_s": 425,
"pcie_tx_avg_mb_s": 288.6200000000000045474735088646411895751953125,
"pcie_tx_max_mb_s": 400,
"vram_used_avg_mb": 128040,
"vram_used_max_mb": 128040,
"power_total_avg_w": 275.56999999999999317878973670303821563720703125,
"power_total_max_w": 304.1299999999999954525264911353588104248046875,
"vram_used_avg_pct": 97.68999999999999772626324556767940521240234375,
"vram_used_max_pct": 97.68999999999999772626324556767940521240234375,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 32314,
"output_seq_len_avg": 743,
"output_seq_len_p50": 743,
"output_seq_len_p90": 743,
"output_seq_len_p99": 743,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.954294999999998339035300887189805507659912109375,
"per_request_avg_tps": 22.03356808664145916054621920920908451080322265625,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1320,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.045486298591655371925668305266299284994602203369140625,
"inter_token_latency_p50": 0.045486298591655371925668305266299284994602203369140625,
"inter_token_latency_p90": 0.045487107642872302848413568199248402379453182220458984375,
"inter_token_latency_p99": 0.045487289679396115948950551910456852056086063385009765625,
"output_tps_per_user_avg": 21.98464222098249365444644354283809661865234375,
"output_tps_per_user_p50": 21.98464222098249365444644354283809661865234375,
"output_tps_per_user_p90": 21.985033255252009354308029287494719028472900390625,
"output_tps_per_user_p99": 21.985121237962648166330836829729378223419189453125,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.000434999999999519104676437564194202423095703125,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.12488501801271922886371612548828125,
"time_to_second_token_p50": 0.12488501801271922886371612548828125,
"time_to_second_token_p90": 0.12488916600705124437808990478515625,
"time_to_second_token_p99": 0.12489009930577595064438156668984447605907917022705078125,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.04554768360999768284091970826921169646084308624267578125,
"chunk_inter_token_latency_p50": 0.04554768360999768284091970826921169646084308624267578125,
"chunk_inter_token_latency_p90": 0.0455484937530516165171690090573974885046482086181640625,
"chunk_inter_token_latency_p99": 0.04554867603523875263515918732082354836165904998779296875
},
{
"ttft_avg": 1.369318482480593957006931304931640625,
"ttft_p50": 1.369271054980345070362091064453125,
"ttft_p90": 1.3701264817907941040431296642054803669452667236328125,
"ttft_p99": 1.3704542331845732672235271820682100951671600341796875,
"wall_time": 35.04095488297753036022186279296875,
"num_errors": 0,
"concurrency": 4,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=4/4, stable=3.0s",
"total_tokens": 2156,
"aggregate_tps": 72.21174344465936201231670565903186798095703125,
"loop_detected": false,
"num_completed": 4,
"request_count": 4,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 32768,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 1.36926535400561988353729248046875,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 602,
"output_tps_per_user": 18.21559904691908826634971774183213710784912109375,
"time_to_second_token": 0.1680467319674789905548095703125,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.054898002389283807878062049212530837394297122955322265625,
"chunk_inter_token_latency_avg": 0.054989499059932618607593468595950980670750141143798828125
},
{
"ttft": 1.3704906500061042606830596923828125,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 602,
"output_tps_per_user": 18.215518631579197261771696503274142742156982421875,
"time_to_second_token": 0.1683624809957109391689300537109375,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.054898244745354519980029550652034231461584568023681640625,
"chunk_inter_token_latency_avg": 0.0549897418199301046826121819321997463703155517578125
},
{
"ttft": 1.3692767559550702571868896484375,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 602,
"output_tps_per_user": 18.21552089400220353354598046280443668365478515625,
"time_to_second_token": 0.168348705046810209751129150390625,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.0548982379268258269267022342319251038134098052978515625,
"chunk_inter_token_latency_avg": 0.0549897349900372034436912827004562132060527801513671875
},
{
"ttft": 1.3682411699555814266204833984375,
"latency": 0,
"completed": false,
"input_tokens": 32314,
"output_tokens": 602,
"output_tps_per_user": 18.215532122379709534243374946527183055877685546875,
"time_to_second_token": 0.168331202003173530101776123046875,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.054898204086576980753608978602642309851944446563720703125,
"chunk_inter_token_latency_avg": 0.054989701093387945152191065290026017464697360992431640625
}
],
"warmup_duration": 5.01100000000000012079226507921703159809112548828125,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 13,
"gpu_count": 2,
"temp_avg_c": 52.4200000000000017053025658242404460906982421875,
"temp_max_c": 54,
"vram_total_mb": 131072,
"cpu_temp_max_c": 65.75,
"cpu_util_avg_pct": 15.9199999999999999289457264239899814128875732421875,
"duration_seconds": 26.559999999999998721023075631819665431976318359375,
"gpu_util_avg_pct": 82.8799999999999954525264911353588104248046875,
"gpu_util_max_pct": 90,
"mem_util_avg_pct": 10.5,
"mem_util_max_pct": 12,
"pcie_rx_avg_mb_s": 386.3799999999999954525264911353588104248046875,
"pcie_rx_max_mb_s": 664,
"pcie_tx_avg_mb_s": 381.1499999999999772626324556767940521240234375,
"pcie_tx_max_mb_s": 455,
"vram_used_avg_mb": 128040,
"vram_used_max_mb": 128040,
"power_total_avg_w": 274.76999999999998181010596454143524169921875,
"power_total_max_w": 314.41000000000002501110429875552654266357421875,
"vram_used_avg_pct": 97.68999999999999772626324556767940521240234375,
"vram_used_max_pct": 97.68999999999999772626324556767940521240234375,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 32314,
"output_seq_len_avg": 602,
"output_seq_len_p50": 602,
"output_seq_len_p90": 602,
"output_seq_len_p99": 602,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.856639999999998735802364535629749298095703125,
"per_request_avg_tps": 18.0529358611648405030791764147579669952392578125,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 2156,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.05489817228701028561932417915159021504223346710205078125,
"inter_token_latency_p50": 0.05489822100670140037070865446366951800882816314697265625,
"inter_token_latency_p90": 0.05489824269979591553347830767961568199098110198974609375,
"inter_token_latency_p99": 0.054898244540798661617042597526960889808833599090576171875,
"output_tps_per_user_avg": 18.2155426737200514253345318138599395751953125,
"output_tps_per_user_p50": 18.21552650819095475753783830441534519195556640625,
"output_tps_per_user_p90": 18.215578969557274291446447023190557956695556640625,
"output_tps_per_user_p99": 18.215597039182906513588022789917886257171630859375,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.001861999999999142119122552685439586639404296875,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.168272280003293417394161224365234375,
"time_to_second_token_p50": 0.16833995352499186992645263671875,
"time_to_second_token_p90": 0.168358348211040731445820028966409154236316680908203125,
"time_to_second_token_p99": 0.1683620677172439072943888049849192611873149871826171875,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.0549896692408219645020750476760440506041049957275390625,
"chunk_inter_token_latency_p50": 0.0549897180417125708284942220416269265115261077880859375,
"chunk_inter_token_latency_p90": 0.05498973977096223431093591216267668642103672027587890625,
"chunk_inter_token_latency_p99": 0.054989741615033317645444554955247440375387668609619140625
},
{
"ttft_avg": 0.741755775990895926952362060546875,
"ttft_p50": 0.741755775990895926952362060546875,
"ttft_p90": 0.741755775990895926952362060546875,
"ttft_p99": 0.741755775990895926952362060546875,
"wall_time": 124.2549326359876431524753570556640625,
"num_errors": 0,
"concurrency": 1,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=1/1, stable=3.0s",
"total_tokens": 1376,
"aggregate_tps": 45.87200742059142299922314123250544071197509765625,
"loop_detected": false,
"num_completed": 1,
"request_count": 1,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 131072,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.741755775990895926952362060546875,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 1581,
"output_tps_per_user": 46.7945750380159921633094199933111667633056640625,
"time_to_second_token": 0.06267721601761877536773681640625,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.0213699985348215736047361446026116027496755123138427734375,
"chunk_inter_token_latency_avg": 0.0213970834505818052750925062355236150324344635009765625
}
],
"warmup_duration": 94.1979999999999932924765744246542453765869140625,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 55.96000000000000085265128291212022304534912109375,
"temp_max_c": 58,
"vram_total_mb": 131072,
"cpu_temp_max_c": 65.75,
"cpu_util_avg_pct": 9.839999999999999857891452847979962825775146484375,
"duration_seconds": 28.666000000000000369482222595252096652984619140625,
"gpu_util_avg_pct": 88.18000000000000682121026329696178436279296875,
"gpu_util_max_pct": 93,
"mem_util_avg_pct": 13.92999999999999971578290569595992565155029296875,
"mem_util_max_pct": 15,
"pcie_rx_avg_mb_s": 478.5,
"pcie_rx_max_mb_s": 622,
"pcie_tx_avg_mb_s": 434.79000000000002046363078989088535308837890625,
"pcie_tx_max_mb_s": 480,
"vram_used_avg_mb": 128066,
"vram_used_max_mb": 128066,
"power_total_avg_w": 327.81999999999999317878973670303821563720703125,
"power_total_max_w": 363.54000000000002046363078989088535308837890625,
"vram_used_avg_pct": 97.7099999999999937472239253111183643341064453125,
"vram_used_max_pct": 97.7099999999999937472239253111183643341064453125,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 128886,
"output_seq_len_avg": 1581,
"output_seq_len_p50": 1581,
"output_seq_len_p90": 1581,
"output_seq_len_p99": 1581,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.99650700000000114187059807591140270233154296875,
"per_request_avg_tps": 45.87200742059142299922314123250544071197509765625,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1376,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.0213699985348215736047361446026116027496755123138427734375,
"inter_token_latency_p50": 0.0213699985348215736047361446026116027496755123138427734375,
"inter_token_latency_p90": 0.0213699985348215736047361446026116027496755123138427734375,
"inter_token_latency_p99": 0.0213699985348215736047361446026116027496755123138427734375,
"output_tps_per_user_avg": 46.7945750380159921633094199933111667633056640625,
"output_tps_per_user_p50": 46.7945750380159921633094199933111667633056640625,
"output_tps_per_user_p90": 46.7945750380159921633094199933111667633056640625,
"output_tps_per_user_p99": 46.7945750380159921633094199933111667633056640625,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.0006360000000000809450284577906131744384765625,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.06267721601761877536773681640625,
"time_to_second_token_p50": 0.06267721601761877536773681640625,
"time_to_second_token_p90": 0.06267721601761877536773681640625,
"time_to_second_token_p99": 0.06267721601761877536773681640625,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.0213970834505818052750925062355236150324344635009765625,
"chunk_inter_token_latency_p50": 0.0213970834505818052750925062355236150324344635009765625,
"chunk_inter_token_latency_p90": 0.0213970834505818052750925062355236150324344635009765625,
"chunk_inter_token_latency_p99": 0.0213970834505818052750925062355236150324344635009765625
},
{
"ttft_avg": 1.4248331150156445801258087158203125,
"ttft_p50": 1.4248331150156445801258087158203125,
"ttft_p90": 1.426356959820259451277024709270335733890533447265625,
"ttft_p99": 1.42669982490129765295705510652624070644378662109375,
"wall_time": 35.520183513988740742206573486328125,
"num_errors": 0,
"concurrency": 2,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=2/2, stable=3.0s",
"total_tokens": 1414,
"aggregate_tps": 47.31649019101207187532054376788437366485595703125,
"loop_detected": false,
"num_completed": 2,
"request_count": 2,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 131072,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 1.4267379210214130580425262451171875,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 797,
"output_tps_per_user": 24.14420940460387754455950926057994365692138671875,
"time_to_second_token": 0.1205797499860636889934539794921875,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.041417798497445000582839469416285282932221889495849609375,
"chunk_inter_token_latency_avg": 0.04146989635719021405346751407705596648156642913818359375
},
{
"ttft": 1.4229283090098761022090911865234375,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 797,
"output_tps_per_user": 24.144259811672402094018252682872116565704345703125,
"time_to_second_token": 0.122187150991521775722503662109375,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.041417712027624710435791399731897399760782718658447265625,
"chunk_inter_token_latency_avg": 0.041469809778602852168294390367009327746927738189697265625
}
],
"warmup_duration": 5.51499999999999968025576890795491635799407958984375,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 52.3599999999999994315658113919198513031005859375,
"temp_max_c": 54,
"vram_total_mb": 131072,
"cpu_temp_max_c": 65.75,
"cpu_util_avg_pct": 21.089999999999999857891452847979962825775146484375,
"duration_seconds": 28.754000000000001335820343228988349437713623046875,
"gpu_util_avg_pct": 80.75,
"gpu_util_max_pct": 91,
"mem_util_avg_pct": 7.92999999999999971578290569595992565155029296875,
"mem_util_max_pct": 9,
"pcie_rx_avg_mb_s": 315.6399999999999863575794734060764312744140625,
"pcie_rx_max_mb_s": 542,
"pcie_tx_avg_mb_s": 270.06999999999999317878973670303821563720703125,
"pcie_tx_max_mb_s": 400,
"vram_used_avg_mb": 128066,
"vram_used_max_mb": 128066,
"power_total_avg_w": 262.73000000000001818989403545856475830078125,
"power_total_max_w": 309.32999999999998408384271897375583648681640625,
"vram_used_avg_pct": 97.7099999999999937472239253111183643341064453125,
"vram_used_max_pct": 97.7099999999999937472239253111183643341064453125,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 128886,
"output_seq_len_avg": 797,
"output_seq_len_p50": 797,
"output_seq_len_p90": 797,
"output_seq_len_p99": 797,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.88387300000000124100552056916058063507080078125,
"per_request_avg_tps": 23.658245095506035937660271883942186832427978515625,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1414,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.0414177552625348555093154345740913413465023040771484375,
"inter_token_latency_p50": 0.0414177552625348555093154345740913413465023040771484375,
"inter_token_latency_p90": 0.04141778985046297434369222401073784567415714263916015625,
"inter_token_latency_p99": 0.041417797632746798652814135266453376971185207366943359375,
"output_tps_per_user_avg": 24.14423460813814159564572037197649478912353515625,
"output_tps_per_user_p50": 24.14423460813814159564572037197649478912353515625,
"output_tps_per_user_p90": 24.144254770965549283801010460592806339263916015625,
"output_tps_per_user_p99": 24.144259307601718234081999980844557285308837890625,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.001069999999998572093318216502666473388671875,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.12138345048879273235797882080078125,
"time_to_second_token_p50": 0.12138345048879273235797882080078125,
"time_to_second_token_p90": 0.1220264108909759614984835707218735478818416595458984375,
"time_to_second_token_p99": 0.12217107698146718874898652984484215267002582550048828125,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.04146985306789653658032790417564683593809604644775390625,
"chunk_inter_token_latency_p50": 0.04146985306789653658032790417564683593809604644775390625,
"chunk_inter_token_latency_p90": 0.04146988769933147855883959209677414037287235260009765625,
"chunk_inter_token_latency_p99": 0.041469895491404340504004721879027783870697021484375
},
{
"ttft_avg": 2.55717788773472420871257781982421875,
"ttft_p50": 2.4377668169909156858921051025390625,
"ttft_p90": 2.773583921673707930466434845584444701671600341796875,
"ttft_p99": 2.90282628804096010100010971655137836933135986328125,
"wall_time": 37.6273691519745625555515289306640625,
"num_errors": 0,
"concurrency": 4,
"underfilled": false,
"ready_reason": "metrics_unavailable, active_streams=4/4, stable=3.0s",
"total_tokens": 1958,
"aggregate_tps": 65.4458973006341722111756098456680774688720703125,
"loop_detected": false,
"num_completed": 4,
"request_count": 4,
"avg_queue_reqs": 0,
"benchmark_mode": "duration",
"context_tokens": 131072,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 2.4359913659864105284214019775390625,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 561,
"output_tps_per_user": 16.47003156468459650341173983179032802581787109375,
"time_to_second_token": 0.4889023060095496475696563720703125,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.06071633779647526985012717659628833644092082977294921875,
"chunk_inter_token_latency_avg": 0.06071633779647526985012717659628833644092082977294921875
},
{
"ttft": 2.4370225140010006725788116455078125,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 561,
"output_tps_per_user": 16.47237714320772283826954662799835205078125,
"time_to_second_token": 0.4840555950067937374114990234375,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.060707692114270427052868939199470332823693752288818359375,
"chunk_inter_token_latency_avg": 0.060707692114270427052868939199470332823693752288818359375
},
{
"ttft": 2.4385111199808306992053985595703125,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 561,
"output_tps_per_user": 16.474854724062456767796902568079531192779541015625,
"time_to_second_token": 0.4789016310242004692554473876953125,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.060698562551780403662160523481361451558768749237060546875,
"chunk_inter_token_latency_avg": 0.060698562551780403662160523481361451558768749237060546875
},
{
"ttft": 2.9171865509706549346446990966796875,
"latency": 0,
"completed": false,
"input_tokens": 128886,
"output_tokens": 553,
"output_tps_per_user": 16.474151582113421454778290353715419769287109375,
"time_to_second_token": 0.185012508998624980449676513671875,
"e2e_output_tps_per_user": 0,
"inter_token_latency_avg": 0.060701153259129649042424858862432301975786685943603515625,
"chunk_inter_token_latency_avg": 0.060701153259129649042424858862432301975786685943603515625
}
],
"warmup_duration": 7.5220000000000002415845301584340631961822509765625,
"aggregate_source": "openai_continuous_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 14,
"gpu_count": 2,
"temp_avg_c": 52.8599999999999994315658113919198513031005859375,
"temp_max_c": 55,
"vram_total_mb": 131072,
"cpu_temp_max_c": 64.8799999999999954525264911353588104248046875,
"cpu_util_avg_pct": 15.980000000000000426325641456060111522674560546875,
"duration_seconds": 28.7530000000000001136868377216160297393798828125,
"gpu_util_avg_pct": 83.1400000000000005684341886080801486968994140625,
"gpu_util_max_pct": 91,
"mem_util_avg_pct": 10.82000000000000028421709430404007434844970703125,
"mem_util_max_pct": 12,
"pcie_rx_avg_mb_s": 414.79000000000002046363078989088535308837890625,
"pcie_rx_max_mb_s": 820,
"pcie_tx_avg_mb_s": 368.93000000000000682121026329696178436279296875,
"pcie_tx_max_mb_s": 573,
"vram_used_avg_mb": 128066,
"vram_used_max_mb": 128066,
"power_total_avg_w": 303.16000000000002501110429875552654266357421875,
"power_total_max_w": 337.1299999999999954525264911353588104248046875,
"vram_used_avg_pct": 97.7099999999999937472239253111183643341064453125,
"vram_used_max_pct": 97.7099999999999937472239253111183643341064453125,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 128886,
"output_seq_len_avg": 559,
"output_seq_len_p50": 561,
"output_seq_len_p90": 561,
"output_seq_len_p99": 561,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 29.91784200000000026875568437390029430389404296875,
"per_request_avg_tps": 16.361474325158543052793902461417019367218017578125,
"request_latency_avg": 0,
"request_latency_p50": 0,
"request_latency_p90": 0,
"request_latency_p99": 0,
"server_engine_steps": 0,
"client_output_tokens": 1958,
"request_count_target": 0,
"server_output_tokens": 0,
"warmup_request_count": 0,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 0,
"inter_token_latency_avg": 0.060705936430413935667171898558081011287868022918701171875,
"inter_token_latency_p50": 0.0607044226867000380476468990309513173997402191162109375,
"inter_token_latency_p90": 0.0607137440918138204803966573308571241796016693115234375,
"inter_token_latency_p99": 0.06071607842600912352537534388829953968524932861328125,
"output_tps_per_user_avg": 16.47285375351705027924253954552114009857177734375,
"output_tps_per_user_p50": 16.4732643626605721465239184908568859100341796875,
"output_tps_per_user_p90": 16.47464378147774510807721526362001895904541015625,
"output_tps_per_user_p99": 16.474833629803985246553565957583487033843994140625,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 30.00104400000000026693669497035443782806396484375,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.40921801025979220867156982421875,
"time_to_second_token_p50": 0.48147861301549710333347320556640625,
"time_to_second_token_p90": 0.487448292708722863419978921228903345763683319091796875,
"time_to_second_token_p99": 0.48875690467946697470580375011195428669452667236328125,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 0,
"e2e_output_tps_per_user_p50": 0,
"e2e_output_tps_per_user_p90": 0,
"e2e_output_tps_per_user_p99": 0,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.060705936430413935667171898558081011287868022918701171875,
"chunk_inter_token_latency_p50": 0.0607044226867000380476468990309513173997402191162109375,
"chunk_inter_token_latency_p90": 0.0607137440918138204803966573308571241796016693115234375,
"chunk_inter_token_latency_p99": 0.06071607842600912352537534388829953968524932861328125
}
],
"metadata": {
"model": "GLM-5.3-Flash-high",
"engine": "openai_proxy",
"version": "0.8.0",
"ignore_eos": true,
"max_tokens": 2048,
"decode_mode": "duration",
"temperature": 0,
"prefill_mode": "skipped",
"skip_prefill": true,
"context_lengths": [
0,
32768,
131072
],
"duration_per_test": 30,
"concurrency_levels": [
1,
2,
4
],
"primary_decode_layer": "sustained_decode",
"input_token_targeting": "upstream estimate fallback: /tokenize returns token_length, not count",
"packaged_from_multiple_original_runs": true
},
"gpu_count": 2,
"gpu_model": "NVIDIA CMP 170HX",
"report_md": "# GLM-5.3-Flash, high effort, on 2x CMP170HX\n\nFour native slots; EXL3 3.05bpw with the Weightless GLP44 steering vector at alpha 2, GPU-resident Q8 KV. 1, 2 and 4 agents at short and nominal 32K/128K context, one 30-second run per cell. Exact completion-token usage, including reasoning; actual input lengths are retained. A separate four-request 512-token completion cohort.\n\nInference speed only, not accuracy or intelligence. Differences between efforts are not separated from run-to-run variation.\n\nFull methods, tables and caveats: https://github.com/alexvnesta/glm53-flash-cmp170hx-tp/blob/fa0adf1899015f692ddce30391ee7e740a0910c9/docs/COMMUNITY_BENCHMARK_2026-10-17.md\n",
"deployment": {
"offload": "none",
"layout_kind": "tp",
"layout_detail": "TP2 on 2x CMP170HX 64GiB, 4 active sequences",
"offload_detail": "Active weights/KV on GPU; inactive RAM/disk retention"
},
"recipe_slug": "glm-5-3-flash-high-effort-on-2x-cmp170hx-exl3-tp2-four-slots",
"source_runs": [
{
"gates": {
"observed_efforts": [
"high"
],
"cpu_psi_avg10_max": 0.7199999999999999733546474089962430298328399658203125,
"public_router_posts": 0,
"peak_active_sequences": 4
},
"label": "high-ctx0",
"sha256": "d332ca477b020e6d7eeebaf861aeae4f5155e98a8539b4921117960ca4a0de91",
"command": [
"/data2/llm-models/glm53-stream-usage-20261017/bench-venv/bin/python",
"/data2/llm-models/glm53-stream-usage-20261017/observe.py",
"/data2/llm-models/glm53-stream-usage-20261017/llm_decode_bench-v0.8.0.py",
"--",
"--host",
"http://***:8014",
"--model",
"GLM-5.3-Flash-high",
"--concurrency",
"1,2,4",
"--contexts",
"0",
"--prefill-contexts",
"0",
"--skip-prefill",
"--kv-budget",
"655360",
"--max-tokens",
"2048",
"--temperature",
"0",
"--duration",
"30",
"--token-targeting",
"exact",
"--decode-warmup-seconds",
"3",
"--display-mode",
"plain",
"--no-resume",
"--output",
"benchmark_results.json"
],
"started": "2026-10-17T12:33:38.818593+00:00",
"finished": "2026-10-17T12:35:47.845138+00:00",
"raw_file": "runs/high-ctx0-attempt1/benchmark_results.json"
},
{
"gates": {
"observed_efforts": [
"high"
],
"cpu_psi_avg10_max": 0.479999999999999982236431605997495353221893310546875,
"public_router_posts": 0,
"peak_active_sequences": 4
},
"label": "high-ctx32768",
"sha256": "bff7ddf8d751a5a2d5d3ab780b2eeb1c864d2d6d25436888f67966d1af3eb617",
"command": [
"/data2/llm-models/glm53-stream-usage-20261017/bench-venv/bin/python",
"/data2/llm-models/glm53-stream-usage-20261017/observe.py",
"/data2/llm-models/glm53-stream-usage-20261017/llm_decode_bench-v0.8.0.py",
"--",
"--host",
"http://***:8014",
"--model",
"GLM-5.3-Flash-high",
"--concurrency",
"1,2,4",
"--contexts",
"32768",
"--prefill-contexts",
"32768",
"--skip-prefill",
"--kv-budget",
"655360",
"--max-tokens",
"2048",
"--temperature",
"0",
"--duration",
"30",
"--token-targeting",
"exact",
"--decode-warmup-seconds",
"3",
"--display-mode",
"plain",
"--no-resume",
"--output",
"benchmark_results.json"
],
"started": "2026-10-17T12:41:06.466454+00:00",
"finished": "2026-10-17T12:44:03.551624+00:00",
"raw_file": "runs/high-ctx32768-attempt1/benchmark_results.json"
},
{
"gates": {
"observed_efforts": [
"high"
],
"cpu_psi_avg10_max": 0.450000000000000011102230246251565404236316680908203125,
"public_router_posts": 0,
"peak_active_sequences": 4
},
"label": "high-ctx131072",
"sha256": "17c2cb7bcdbec1ff67eb5f1b109d2e41dae4a23171a28eb3a65be1de4ed4d0e4",
"command": [
"/data2/llm-models/glm53-stream-usage-20261017/bench-venv/bin/python",
"/data2/llm-models/glm53-stream-usage-20261017/observe.py",
"/data2/llm-models/glm53-stream-usage-20261017/llm_decode_bench-v0.8.0.py",
"--",
"--host",
"http://***:8014",
"--model",
"GLM-5.3-Flash-high",
"--concurrency",
"1,2,4",
"--contexts",
"131072",
"--prefill-contexts",
"131072",
"--skip-prefill",
"--kv-budget",
"655360",
"--max-tokens",
"2048",
"--temperature",
"0",
"--duration",
"30",
"--token-targeting",
"exact",
"--decode-warmup-seconds",
"3",
"--display-mode",
"plain",
"--no-resume",
"--output",
"benchmark_results.json"
],
"started": "2026-10-17T12:52:30.383943+00:00",
"finished": "2026-10-17T12:57:46.022592+00:00",
"raw_file": "runs/high-ctx131072-attempt1/benchmark_results.json"
},
{
"gates": {
"observed_efforts": [
"high"
],
"cpu_psi_avg10_max": 0.179999999999999993338661852249060757458209991455078125,
"public_router_posts": 0,
"peak_active_sequences": 4
},
"label": "high-finite-ctx0",
"sha256": "965f1d7474c04677c677930f09340499c30092e9f57577c094d40d87705d6148",
"command": [
"/data2/llm-models/glm53-stream-usage-20261017/bench-venv/bin/python",
"/data2/llm-models/glm53-stream-usage-20261017/observe.py",
"/data2/llm-models/glm53-stream-usage-20261017/llm_decode_bench-v0.8.0.py",
"--",
"--host",
"http://***:8014",
"--model",
"GLM-5.3-Flash-high",
"--concurrency",
"1,2,4",
"--contexts",
"0",
"--prefill-contexts",
"0",
"--skip-prefill",
"--kv-budget",
"655360",
"--max-tokens",
"512",
"--temperature",
"0",
"--duration",
"30",
"--token-targeting",
"exact",
"--decode-warmup-seconds",
"3",
"--display-mode",
"plain",
"--no-resume",
"--output",
"benchmark_results.json",
"--request-count",
"4",
"--warmup-request-count",
"1"
],
"started": "2026-10-17T13:14:24.900552+00:00",
"finished": "2026-10-17T13:17:03.842977+00:00",
"raw_file": "runs/high-finite-ctx0-attempt1/benchmark_results.json"
}
],
"tool_commit": "497350a81a77d7d11b4e418b2a26c30c3065949a",
"agent_audits": [
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 1,
"benchmark_mode": "duration",
"context_tokens": 0,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
1293
],
"tokens_per_second": [
43.16218167036803521341425948776304721832275390625
],
"requested_duration": 30,
"measurement_seconds": 29.9567804490216076374053955078125,
"request_count_target": 0,
"aggregate_tokens_per_second": 43.16218167036803521341425948776304721832275390625,
"measurement_start_monotonic": 508703.2553281440050341188907623291015625,
"reported_measurement_seconds_rounded": 29.956779999999998409521140274591743946075439453125
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 2,
"benchmark_mode": "duration",
"context_tokens": 0,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
896,
895
],
"tokens_per_second": [
29.867225432642218407863765605725347995758056640625,
29.8338914756861441901492071337997913360595703125
],
"requested_duration": 30,
"measurement_seconds": 29.9994387500337325036525726318359375,
"request_count_target": 0,
"aggregate_tokens_per_second": 59.70111690832836615072665154002606868743896484375,
"measurement_start_monotonic": 508740.79326988197863101959228515625,
"reported_measurement_seconds_rounded": 29.999438999999998856083038845099508762359619140625
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 4,
"benchmark_mode": "duration",
"context_tokens": 0,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
721,
721,
721,
721
],
"tokens_per_second": [
24.067186037454149527547997422516345977783203125,
24.067186037454149527547997422516345977783203125,
24.067186037454149527547997422516345977783203125,
24.067186037454149527547997422516345977783203125
],
"requested_duration": 30,
"measurement_seconds": 29.957802249002270400524139404296875,
"request_count_target": 0,
"aggregate_tokens_per_second": 96.2687441498165981101919896900653839111328125,
"measurement_start_monotonic": 508778.3189110929961316287517547607421875,
"reported_measurement_seconds_rounded": 29.9578020000000009304130799137055873870849609375
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 1,
"benchmark_mode": "duration",
"context_tokens": 32768,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
1279
],
"tokens_per_second": [
42.67706541990319379920038045383989810943603515625
],
"requested_duration": 30,
"measurement_seconds": 29.9692583690048195421695709228515625,
"request_count_target": 0,
"aggregate_tokens_per_second": 42.67706541990319379920038045383989810943603515625,
"measurement_start_monotonic": 509200.2856404469930566847324371337890625,
"reported_measurement_seconds_rounded": 29.969257999999999952933649183250963687896728515625
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 2,
"benchmark_mode": "duration",
"context_tokens": 32768,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
660,
660
],
"tokens_per_second": [
22.03356808664145916054621920920908451080322265625,
22.03356808664145916054621920920908451080322265625
],
"requested_duration": 30,
"measurement_seconds": 29.954295073985118591508580720983445644378662109375,
"request_count_target": 0,
"aggregate_tokens_per_second": 44.0671361732829183210924384184181690216064453125,
"measurement_start_monotonic": 509237.349865500000305473804473876953125,
"reported_measurement_seconds_rounded": 29.954294999999998339035300887189805507659912109375
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 4,
"benchmark_mode": "duration",
"context_tokens": 32768,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
539,
539,
539,
539
],
"tokens_per_second": [
18.0529358611648405030791764147579669952392578125,
18.0529358611648405030791764147579669952392578125,
18.0529358611648405030791764147579669952392578125,
18.0529358611648405030791764147579669952392578125
],
"requested_duration": 30,
"measurement_seconds": 29.85663961502723395824432373046875,
"request_count_target": 0,
"aggregate_tokens_per_second": 72.21174344465936201231670565903186798095703125,
"measurement_start_monotonic": 509274.463826533989049494266510009765625,
"reported_measurement_seconds_rounded": 29.856639999999998735802364535629749298095703125
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 1,
"benchmark_mode": "duration",
"context_tokens": 131072,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
1376
],
"tokens_per_second": [
45.87200742059142299922314123250544071197509765625
],
"requested_duration": 30,
"measurement_seconds": 29.9965071810293011367321014404296875,
"request_count_target": 0,
"aggregate_tokens_per_second": 45.87200742059142299922314123250544071197509765625,
"measurement_start_monotonic": 510019.3238294449984095990657806396484375,
"reported_measurement_seconds_rounded": 29.99650700000000114187059807591140270233154296875
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 2,
"benchmark_mode": "duration",
"context_tokens": 131072,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
707,
707
],
"tokens_per_second": [
23.658245095506035937660271883942186832427978515625,
23.658245095506035937660271883942186832427978515625
],
"requested_duration": 30,
"measurement_seconds": 29.8838733450393192470073699951171875,
"request_count_target": 0,
"aggregate_tokens_per_second": 47.31649019101207187532054376788437366485595703125,
"measurement_start_monotonic": 510056.9056146019720472395420074462890625,
"reported_measurement_seconds_rounded": 29.88387300000000124100552056916058063507080078125
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 4,
"benchmark_mode": "duration",
"context_tokens": 131072,
"aggregate_source": "openai_continuous_usage",
"completion_tokens": [
491,
491,
491,
485
],
"tokens_per_second": [
16.41161163156863267431617714464664459228515625,
16.41161163156863267431617714464664459228515625,
16.41161163156863267431617714464664459228515625,
16.21106240592828129365443601273000240325927734375
],
"requested_duration": 30,
"measurement_seconds": 29.917841770977243953666402376256883144378662109375,
"request_count_target": 0,
"aggregate_tokens_per_second": 65.4458973006341722111756098456680774688720703125,
"measurement_start_monotonic": 510096.443296227022074162960052490234375,
"reported_measurement_seconds_rounded": 29.91784200000000026875568437390029430389404296875
}
],
"slot_ceiling": 4,
"tool_version": "0.8.0",
"burst_results": [
{
"ttft_avg": 0.28727926322608254849910736083984375,
"ttft_p50": 0.2881484734825789928436279296875,
"ttft_p90": 0.290411994286114361063511069005471654236316680908203125,
"ttft_p99": 0.290923905909876323594431823948980309069156646728515625,
"wall_time": 46.254615462035872042179107666015625,
"num_errors": 0,
"concurrency": 1,
"underfilled": false,
"ready_reason": "request_count=4, warmup_request_count=1",
"total_tokens": 2048,
"aggregate_tps": 44.27666254583663629773582215420901775360107421875,
"loop_detected": false,
"num_completed": 4,
"request_count": 4,
"avg_queue_reqs": 0,
"benchmark_mode": "request-count",
"context_tokens": 0,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.2818393209599889814853668212890625,
"latency": 11.5104674839531071484088897705078125,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 45.508675911464791852267808280885219573974609375,
"time_to_second_token": 0.0573636970366351306438446044921875,
"e2e_output_tps_per_user": 44.481251583724628062554984353482723236083984375,
"inter_token_latency_avg": 0.021973832021512949841390849314848310314118862152099609375,
"chunk_inter_token_latency_avg": 0.0220169179666531746042412720498759881593286991119384765625
},
{
"ttft": 0.289084816002286970615386962890625,
"latency": 11.5199133050045929849147796630859375,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 45.4997599242471295610812376253306865692138671875,
"time_to_second_token": 0.057748837978579103946685791015625,
"e2e_output_tps_per_user": 44.444778918394462152718915604054927825927734375,
"inter_token_latency_avg": 0.0219781379432530434014925191377187729813158512115478515625,
"chunk_inter_token_latency_avg": 0.0220212323313770698407498827009476372040808200836181640625
},
{
"ttft": 0.287212130962871015071868896484375,
"latency": 11.50121388700790703296661376953125,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 45.56803281438221375765351695008575916290283203125,
"time_to_second_token": 0.058015484013594686985015869140625,
"e2e_output_tps_per_user": 44.51704011681493255991881596855819225311279296875,
"inter_token_latency_avg": 0.0219452089159394049533435833154726424254477024078369140625,
"chunk_inter_token_latency_avg": 0.0219882387373432075516888062338693998754024505615234375
},
{
"ttft": 0.2909807849791832268238067626953125,
"latency": 11.5643010989879257977008819580078125,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 45.32826050946215445947018451988697052001953125,
"time_to_second_token": 0.056946145021356642246246337890625,
"e2e_output_tps_per_user": 44.27418445934521429308006190694868564605712890625,
"inter_token_latency_avg": 0.0220612921996257184409362395172138349153101444244384765625,
"chunk_inter_token_latency_avg": 0.0221045496353112595755430191957202623598277568817138671875
}
],
"warmup_duration": 0,
"aggregate_source": "openai_completed_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 21,
"gpu_count": 2,
"temp_avg_c": 55.18999999999999772626324556767940521240234375,
"temp_max_c": 57,
"vram_total_mb": 131072,
"cpu_temp_max_c": 67.6200000000000045474735088646411895751953125,
"cpu_util_avg_pct": 10.17999999999999971578290569595992565155029296875,
"duration_seconds": 44.18500000000000227373675443232059478759765625,
"gpu_util_avg_pct": 86.7900000000000062527760746888816356658935546875,
"gpu_util_max_pct": 93,
"mem_util_avg_pct": 14.28999999999999914734871708787977695465087890625,
"mem_util_max_pct": 16,
"pcie_rx_avg_mb_s": 483.67000000000001591615728102624416351318359375,
"pcie_rx_max_mb_s": 583,
"pcie_tx_avg_mb_s": 465.48000000000001818989403545856475830078125,
"pcie_tx_max_mb_s": 538,
"vram_used_avg_mb": 128774,
"vram_used_max_mb": 128774,
"power_total_avg_w": 331.779999999999972715158946812152862548828125,
"power_total_max_w": 371.54000000000002046363078989088535308837890625,
"vram_used_avg_pct": 98.25,
"vram_used_max_pct": 98.25,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 78,
"output_seq_len_avg": 512,
"output_seq_len_p50": 512,
"output_seq_len_p90": 512,
"output_seq_len_p99": 512,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 46.25461500000000114596332423388957977294921875,
"per_request_avg_tps": 44.27666254583663629773582215420901775360107421875,
"request_latency_avg": 11.523973943738383240997791290283203125,
"request_latency_p50": 11.515190394478850066661834716796875,
"request_latency_p90": 11.550984760792925243322315509431064128875732421875,
"request_latency_p99": 11.562969465168425386991657433100044727325439453125,
"server_engine_steps": 0,
"client_output_tokens": 2048,
"request_count_target": 4,
"server_output_tokens": 0,
"warmup_request_count": 1,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 4,
"inter_token_latency_avg": 0.02198961777008277829192905983290984295308589935302734375,
"inter_token_latency_p50": 0.02197598498238299835616516020309063605964183807373046875,
"inter_token_latency_p90": 0.0220363459227139145413243426219196408055722713470458984375,
"inter_token_latency_p99": 0.022058797571934539438753830609130091033875942230224609375,
"output_tps_per_user_avg": 45.47618228988907418397502624429762363433837890625,
"output_tps_per_user_p50": 45.50421791785596070667452295310795307159423828125,
"output_tps_per_user_p90": 45.5502257435069850544095970690250396728515625,
"output_tps_per_user_p99": 45.56625210729469444004280376248061656951904296875,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 0,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.057518541012541390955448150634765625,
"time_to_second_token_p50": 0.05755626750760711729526519775390625,
"time_to_second_token_p90": 0.05793549020309000929795928414023364894092082977294921875,
"time_to_second_token_p99": 0.058007484632544216440752649077694513835012912750244140625,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 44.42931376956980926706819445826113224029541015625,
"e2e_output_tps_per_user_p50": 44.463015251059545107636949978768825531005859375,
"e2e_output_tps_per_user_p90": 44.506303556887843342337873764336109161376953125,
"e2e_output_tps_per_user_p99": 44.51596646082222008544704294763505458831787109375,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.0220327346676711770256940070566997746936976909637451171875,
"chunk_inter_token_latency_p50": 0.0220190751490151222224955773754118126817047595977783203125,
"chunk_inter_token_latency_p90": 0.022079554444131004042883859028734150342643260955810546875,
"chunk_inter_token_latency_p99": 0.0221020501161932336753324079836602322757244110107421875
},
{
"ttft_avg": 0.504481854499317705631256103515625,
"ttft_p50": 0.5466034740093164145946502685546875,
"ttft_p90": 0.60135851099621506765657841242500580847263336181640625,
"ttft_p99": 0.62230600738897923207559870206750929355621337890625,
"wall_time": 34.20735388598404824733734130859375,
"num_errors": 0,
"concurrency": 2,
"underfilled": false,
"ready_reason": "request_count=4, warmup_request_count=1",
"total_tokens": 2048,
"aggregate_tps": 59.87016729870875764163429266773164272308349609375,
"loop_detected": false,
"num_completed": 4,
"request_count": 4,
"avg_queue_reqs": 0,
"benchmark_mode": "request-count",
"context_tokens": 0,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 0.54615676100365817546844482421875,
"latency": 15.781528602004982531070709228515625,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 33.5403694332422190882425638847053050994873046875,
"time_to_second_token": 0.10182923800311982631683349609375,
"e2e_output_tps_per_user": 32.442991608237008449577842839062213897705078125,
"inter_token_latency_avg": 0.0298148176927618864529012654429607209749519824981689453125,
"chunk_inter_token_latency_avg": 0.0298148176927618864529012654429607209749519824981689453125
},
{
"ttft": 0.300086962990462779998779296875,
"latency": 18.211051687016151845455169677734375,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 28.53000985003040312903976882807910442352294921875,
"time_to_second_token": 0.323406200041063129901885986328125,
"e2e_output_tps_per_user": 28.114795828350661821559697273187339305877685546875,
"inter_token_latency_avg": 0.035050811593005264110711749481197330169379711151123046875,
"chunk_inter_token_latency_avg": 0.03511953867456017730486195205230615101754665374755859375
},
{
"ttft": 0.547050187014974653720855712890625,
"latency": 15.78148750402033329010009765625,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 33.542426895517763796306098811328411102294921875,
"time_to_second_token": 0.102033194969408214092254638671875,
"e2e_output_tps_per_user": 32.443076095936334013458690606057643890380859375,
"inter_token_latency_avg": 0.0298129888786797629662839170805455069057643413543701171875,
"chunk_inter_token_latency_avg": 0.0298129888786797629662839170805455069057643413543701171875
},
{
"ttft": 0.624633506988175213336944580078125,
"latency": 17.595001279958523809909820556640625,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 30.11130971562668179331012652255594730377197265625,
"time_to_second_token": 0.105376182007603347301483154296875,
"e2e_output_tps_per_user": 29.0991737854086096604078193195164203643798828125,
"inter_token_latency_avg": 0.03321011305865038953921697384430444799363613128662109375,
"chunk_inter_token_latency_avg": 0.03321011305865038953921697384430444799363613128662109375
}
],
"warmup_duration": 0,
"aggregate_source": "openai_completed_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 15,
"gpu_count": 2,
"temp_avg_c": 54.5,
"temp_max_c": 56,
"vram_total_mb": 131072,
"cpu_temp_max_c": 67.75,
"cpu_util_avg_pct": 22.5799999999999982946974341757595539093017578125,
"duration_seconds": 30.97500000000000142108547152020037174224853515625,
"gpu_util_avg_pct": 80.2699999999999960209606797434389591217041015625,
"gpu_util_max_pct": 89,
"mem_util_avg_pct": 11.5999999999999996447286321199499070644378662109375,
"mem_util_max_pct": 14,
"pcie_rx_avg_mb_s": 380.80000000000001136868377216160297393798828125,
"pcie_rx_max_mb_s": 516,
"pcie_tx_avg_mb_s": 354.470000000000027284841053187847137451171875,
"pcie_tx_max_mb_s": 465,
"vram_used_avg_mb": 128774,
"vram_used_max_mb": 128774,
"power_total_avg_w": 308.029999999999972715158946812152862548828125,
"power_total_max_w": 365.490000000000009094947017729282379150390625,
"vram_used_avg_pct": 98.25,
"vram_used_max_pct": 98.25,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 78,
"output_seq_len_avg": 512,
"output_seq_len_p50": 512,
"output_seq_len_p90": 512,
"output_seq_len_p99": 512,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 34.20735400000000225873009185306727886199951171875,
"per_request_avg_tps": 29.935083649354378820817146333865821361541748046875,
"request_latency_avg": 16.84226726824999786913394927978515625,
"request_latency_p50": 16.688264940981753170490264892578125,
"request_latency_p90": 18.026236564898862724248829181306064128875732421875,
"request_latency_p99": 18.19257017480442328860590350814163684844970703125,
"server_engine_steps": 0,
"client_output_tokens": 2048,
"request_count_target": 4,
"server_output_tokens": 0,
"warmup_request_count": 1,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 4,
"inter_token_latency_avg": 0.0319721828057743240325550004854449070990085601806640625,
"inter_token_latency_p50": 0.03151246537570613626133564366682549007236957550048828125,
"inter_token_latency_p90": 0.0344986020326988052087102687437436543405055999755859375,
"inter_token_latency_p99": 0.03499559063697461891440099179817480035126209259033203125,
"output_tps_per_user_avg": 31.431028973604266951724639511667191982269287109375,
"output_tps_per_user_p50": 31.825839574434450440776345203630626201629638671875,
"output_tps_per_user_p90": 33.5418096568350989628015668131411075592041015625,
"output_tps_per_user_p99": 33.54236517164950015512658865191042423248291015625,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 0,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.15816120375529862940311431884765625,
"time_to_second_token_p50": 0.103704688488505780696868896484375,
"time_to_second_token_p90": 0.25799719463102521732622562922188080847263336181640625,
"time_to_second_token_p99": 0.31686529950005926092870822685654275119304656982421875,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 30.5250093294831543744294322095811367034912109375,
"e2e_output_tps_per_user_p50": 30.77108269682280905499283107928931713104248046875,
"e2e_output_tps_per_user_p90": 32.443050749626536344294436275959014892578125,
"e2e_output_tps_per_user_p99": 32.4430735613053542465422651730477809906005859375,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.0319893645761630540658160271050292067229747772216796875,
"chunk_inter_token_latency_p50": 0.03151246537570613626133564366682549007236957550048828125,
"chunk_inter_token_latency_p90": 0.034546710989787239587389677808459964580833911895751953125,
"chunk_inter_token_latency_p99": 0.035062255906082882839225334237198694609105587005615234375
},
{
"ttft_avg": 1.023162016019341535866260528564453125,
"ttft_p50": 1.0236347835161723196506500244140625,
"ttft_p90": 1.0243690497183706167305672352085821330547332763671875,
"ttft_p99": 1.02464792489365219552155394922010600566864013671875,
"wall_time": 21.35422313003800809383392333984375,
"num_errors": 0,
"concurrency": 4,
"underfilled": false,
"ready_reason": "request_count=4, warmup_request_count=1",
"total_tokens": 2048,
"aggregate_tps": 95.906087874448218144607380963861942291259765625,
"loop_detected": false,
"num_completed": 4,
"request_count": 4,
"avg_queue_reqs": 0,
"benchmark_mode": "request-count",
"context_tokens": 0,
"failure_reason": "",
"max_queue_reqs": 0,
"queue_fraction": 0,
"timeout_reason": "",
"accept_norm_tps": 0,
"request_samples": [
{
"ttft": 1.0246789110242389142513275146484375,
"latency": 21.3032604469917714595794677734375,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 25.199001177358194780708799953572452068328857421875,
"time_to_second_token": 0.1187360369949601590633392333984375,
"e2e_output_tps_per_user": 24.033879756293334395422789384610950946807861328125,
"inter_token_latency_avg": 0.03968411259484840025724139422891312278807163238525390625,
"chunk_inter_token_latency_avg": 0.03968411259484840025724139422891312278807163238525390625
},
{
"ttft": 1.02069958602078258991241455078125,
"latency": 21.302277431008405983448028564453125,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 25.195278390349113095680877449922263622283935546875,
"time_to_second_token": 0.1192357469699345529079437255859375,
"e2e_output_tps_per_user": 24.034988824937247642310467199422419071197509765625,
"inter_token_latency_avg": 0.039689976213283018058319839838077314198017120361328125,
"chunk_inter_token_latency_avg": 0.039689976213283018058319839838077314198017120361328125
},
{
"ttft": 1.023646040004678070545196533203125,
"latency": 21.302074466017074882984161376953125,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 25.1991914395352551991891232319176197052001953125,
"time_to_second_token": 0.118750295019708573818206787109375,
"e2e_output_tps_per_user": 24.035217828985953048004375887103378772735595703125,
"inter_token_latency_avg": 0.03968381296675616010105613895575515925884246826171875,
"chunk_inter_token_latency_avg": 0.03968381296675616010105613895575515925884246826171875
},
{
"ttft": 1.023623527027666568756103515625,
"latency": 21.3021810670034028589725494384765625,
"completed": true,
"input_tokens": 78,
"output_tokens": 512,
"output_tps_per_user": 25.19903099580185568129309103824198246002197265625,
"time_to_second_token": 0.1187118520028889179229736328125,
"e2e_output_tps_per_user": 24.035097551258562731391066336072981357574462890625,
"inter_token_latency_avg": 0.03968406563596034442031879052592557854950428009033203125,
"chunk_inter_token_latency_avg": 0.03968406563596034442031879052592557854950428009033203125
}
],
"warmup_duration": 0,
"aggregate_source": "openai_completed_usage",
"avg_running_reqs": 0,
"capacity_limited": false,
"hardware_summary": {
"samples": 9,
"gpu_count": 2,
"temp_avg_c": 54.1099999999999994315658113919198513031005859375,
"temp_max_c": 56,
"vram_total_mb": 131072,
"cpu_temp_max_c": 70.75,
"cpu_util_avg_pct": 20.129999999999999005240169935859739780426025390625,
"duration_seconds": 17.669000000000000483169060316868126392364501953125,
"gpu_util_avg_pct": 80.06000000000000227373675443232059478759765625,
"gpu_util_max_pct": 89,
"mem_util_avg_pct": 8.7799999999999993605115378159098327159881591796875,
"mem_util_max_pct": 10,
"pcie_rx_avg_mb_s": 722.44000000000005456968210637569427490234375,
"pcie_rx_max_mb_s": 1233,
"pcie_tx_avg_mb_s": 458.67000000000001591615728102624416351318359375,
"pcie_tx_max_mb_s": 662,
"vram_used_avg_mb": 128774,
"vram_used_max_mb": 128774,
"power_total_avg_w": 301.94999999999998863131622783839702606201171875,
"power_total_max_w": 341.8600000000000136424205265939235687255859375,
"vram_used_avg_pct": 98.25,
"vram_used_max_pct": 98.25,
"power_limit_total_w": 360
},
"loop_diagnostics": [],
"max_running_reqs": 0,
"warmup_timed_out": false,
"input_seq_len_avg": 78,
"output_seq_len_avg": 512,
"output_seq_len_p50": 512,
"output_seq_len_p90": 512,
"output_seq_len_p99": 512,
"server_spec_drafts": 0,
"server_steps_per_s": 0,
"server_utilization": 0,
"accept_norm_ref_len": 0,
"measurement_seconds": 21.354223000000001064790922100655734539031982421875,
"per_request_avg_tps": 23.97652196861205453615184524096548557281494140625,
"request_latency_avg": 21.302448352755163796246051788330078125,
"request_latency_p50": 21.30222924900590442121028900146484375,
"request_latency_p90": 21.302965542196762527282771770842373371124267578125,
"request_latency_p99": 21.303230956512269500535694533027708530426025390625,
"server_engine_steps": 0,
"client_output_tokens": 2048,
"request_count_target": 4,
"server_output_tokens": 0,
"warmup_request_count": 1,
"effective_concurrency": 0,
"server_gen_throughput": 0,
"loop_detection_enabled": true,
"server_spec_pos_accept": [],
"completed_request_count": 4,
"inter_token_latency_avg": 0.039685491852711980709234040887167793698608875274658203125,
"inter_token_latency_p50": 0.03968408911540437233878009237741935066878795623779296875,
"inter_token_latency_p90": 0.039688217127752632717996306155328056775033473968505859375,
"inter_token_latency_p99": 0.039689800304729981605955657641970901750028133392333984375,
"output_tps_per_user_avg": 25.198125500761104689217972918413579463958740234375,
"output_tps_per_user_p50": 25.19901608658002345464410609565675258636474609375,
"output_tps_per_user_p90": 25.199143306415233922734842053614556789398193359375,
"output_tps_per_user_p99": 25.19918662622325200572959147393703460693359375,
"server_spec_accept_rate": 0,
"loop_diagnostics_omitted": 0,
"measurement_wall_seconds": 0,
"server_spec_draft_tokens": 0,
"time_to_second_token_avg": 0.1188584827468730509281158447265625,
"time_to_second_token_p50": 0.11874316600733436644077301025390625,
"time_to_second_token_p90": 0.11909011138486676195658020560586010105907917022705078125,
"time_to_second_token_p99": 0.11922118341142777520058615436937543563544750213623046875,
"server_spec_accept_length": 0,
"e2e_output_tps_per_user_avg": 24.034795990368774454282174701802432537078857421875,
"e2e_output_tps_per_user_p50": 24.035043188097905186850766767747700214385986328125,
"e2e_output_tps_per_user_p90": 24.03518174566773524247764726169407367706298828125,
"e2e_output_tps_per_user_p99": 24.03521422065413304380854242481291294097900390625,
"server_accept_len_effective": 0,
"server_spec_accepted_tokens": 0,
"chunk_inter_token_latency_avg": 0.039685491852711980709234040887167793698608875274658203125,
"chunk_inter_token_latency_p50": 0.03968408911540437233878009237741935066878795623779296875,
"chunk_inter_token_latency_p90": 0.039688217127752632717996306155328056775033473968505859375,
"chunk_inter_token_latency_p99": 0.039689800304729981605955657641970901750028133392333984375
}
],
"engine_version": "0f685aa + row32 four-slot profile",
"model_revision": "332ab457b709b7ba30dd9a448be5de03b80a7ac9",
"profile_digest": "7c6da98c10332410ccf838449ab87fe86c6907c351bdd72958685e034f593031",
"run_conditions": {
"note": "Dedicated quiet intervals on the shared service; yield to public router traffic; CPU PSI avg10 <2%. Effort via model alias. Q8 GPU KV; Weightless GLP-44 alpha2. One run per cell. No cold-prefill claim.",
"via_form": true,
"shared_server": true,
"repeats_per_cell": 1,
"duration_per_cell": 30
},
"reasoning_effort": "high",
"burst_agent_audits": [
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 1,
"benchmark_mode": "request-count",
"context_tokens": 0,
"aggregate_source": "openai_completed_usage",
"completion_tokens": [
2048
],
"tokens_per_second": [
44.27666254583663629773582215420901775360107421875
],
"requested_duration": 30,
"measurement_seconds": 46.254615462035872042179107666015625,
"request_count_target": 4,
"aggregate_tokens_per_second": 44.27666254583663629773582215420901775360107421875,
"measurement_start_monotonic": 511155.12964509497396647930145263671875,
"reported_measurement_seconds_rounded": 46.25461500000000114596332423388957977294921875
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 2,
"benchmark_mode": "request-count",
"context_tokens": 0,
"aggregate_source": "openai_completed_usage",
"completion_tokens": [
1024,
1024
],
"tokens_per_second": [
29.935083649354378820817146333865821361541748046875,
29.935083649354378820817146333865821361541748046875
],
"requested_duration": 30,
"measurement_seconds": 34.20735388598404824733734130859375,
"request_count_target": 4,
"aggregate_tokens_per_second": 59.87016729870875764163429266773164272308349609375,
"measurement_start_monotonic": 511215.19938916899263858795166015625,
"reported_measurement_seconds_rounded": 34.20735400000000225873009185306727886199951171875
},
{
"source": "upstream shared_usage_token_count increments, common upstream window",
"concurrency": 4,
"benchmark_mode": "request-count",
"context_tokens": 0,
"aggregate_source": "openai_completed_usage",
"completion_tokens": [
512,
512,
512,
512
],
"tokens_per_second": [
23.97652196861205453615184524096548557281494140625,
23.97652196861205453615184524096548557281494140625,
23.97652196861205453615184524096548557281494140625,
23.97652196861205453615184524096548557281494140625
],
"requested_duration": 30,
"measurement_seconds": 21.35422313003800809383392333984375,
"request_count_target": 4,
"aggregate_tokens_per_second": 95.906087874448218144607380963861942291259765625,
"measurement_start_monotonic": 511263.2180461179814301431179046630859375,
"reported_measurement_seconds_rounded": 21.354223000000001064790922100655734539031982421875
}
],
"engine_source_commit": "fa0adf1899015f692ddce30391ee7e740a0910c9"
}Run file preserved verbatim in the corpus (unknown keys never dropped) · derived rows regenerate from it on import.