[{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":271,"date":"2026-01-28","num_gpus":18,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1450622.0190386316","tput_per_gpu":2756.1818361734,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":12393.6815508794,"output_tput_per_gpu":1551.4943718351,"p50_itl":2.3075432074256,"mean_itl":2.0389398121142,"p50_e2el":9.5533172194846,"p50_tpot":0.0097829802084,"p50_ttft":0.5528178260429001,"mean_e2el":9.4291409960486,"mean_tpot":0.0093585429191,"mean_ttft":0.8114229454576,"p50_intvty":102.2183402906,"mean_intvty":106.8542409481},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":2596000.678832117,"dataset_name":"random","tput_per_gpu":3556.52093,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":46.90361814,"input_tput_per_gpu":1746.560356,"total_input_tokens":327680,"output_tput_per_gpu":1746.560356,"total_output_tokens":327680,"tput_requests_per_sec":6.822501391,"p50_e2el":8.250320349999999,"p50_tpot":"0.0077816479999999995","p50_ttft":0.289694052,"p50_intvty":117.0478049},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"109039.76233576643","dataset_name":"random","tput_per_gpu":149.3844744,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":15.37894088,"input_tput_per_gpu":74.54186922,"total_input_tokens":9171,"output_tput_per_gpu":74.84260515,"total_output_tokens":9208,"tput_requests_per_sec":0.650239836,"p50_e2el":3.341955062,"p50_tpot":0.003241737,"p50_ttft":0.025658196,"p50_intvty":308.4766055},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"2869710.7058479534","tput_per_gpu":4907.205307,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":17.62566,"input_tput_per_gpu":4204.834,"kv_cache_precision":"FP8","total_input_tokens":74113,"output_tput_per_gpu":527.8099,"total_output_tokens":9303,"tput_requests_per_sec":0.567354,"p50_e2el":1.631976,"p50_tpot":0.001501,"p50_ttft":0.096344,"p50_intvty":616.7399},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":850542.0753173723,"tput_per_gpu":1165.2426431848,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":142.6643198006},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"12760691.631744854","tput_per_gpu":21820.7826902837,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":24.4736869747},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1234885.4101906433","tput_per_gpu":2111.654051426,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":140.2238355375},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":398164.2956880702,"tput_per_gpu":680.8609456266,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":185.6194930276},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":41039.0700010219,"tput_per_gpu":56.2235259014,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":178.7958325421},{"hw":"H100","tp":2,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":367945.6250881752,"tput_per_gpu":504.0855063708,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":85.6864643889},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":64,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":908740.3263671533,"tput_per_gpu":1244.974247123,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":19.1234962726},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":52,"date":"2026-01-28","num_gpus":18,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1178205.6460119998","tput_per_gpu":2238.5907274228,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":17900.3846960277,"output_tput_per_gpu":280.8664813472,"p50_itl":1.7532332011033,"mean_itl":1.5207547109468,"p50_e2el":9.9980011510197,"p50_tpot":0.007056075616400001,"p50_ttft":4.0460016161669,"mean_e2el":"10.103149035872601","mean_tpot":0.0065404267381,"mean_ttft":4.0581494332291,"p50_intvty":141.7218372313,"mean_intvty":152.8952222906},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":25,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":135259.0017931579,"tput_per_gpu":256.992103407,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":2703.2809379655,"output_tput_per_gpu":134.6776616791,"p50_itl":"1.0376697224565001","mean_itl":0.9267604010714,"p50_e2el":4.1884287485154,"p50_tpot":0.0044421424412,"p50_ttft":"0.12816249742170002","mean_e2el":4.1128980633141,"mean_tpot":0.0042683287202,"mean_ttft":0.1982623645114,"p50_intvty":225.1165993073,"mean_intvty":234.2837362217},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"16892674.175438598","tput_per_gpu":28886.47284,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":81.9714,"input_tput_per_gpu":14479.36,"kv_cache_precision":"FP8","total_input_tokens":2373787,"output_tput_per_gpu":14407.11,"total_output_tokens":2361942,"tput_requests_per_sec":31.23041,"p50_e2el":8.583983,"p50_tpot":0.008317,"p50_ttft":0.075616,"p50_intvty":120.2346},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":1024,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"17745122.456125207","tput_per_gpu":30344.1593999741,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":9.152319081},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"20318.885900877194","tput_per_gpu":34.7452948905,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":216.8267391944},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":3817669.272998304,"tput_per_gpu":6528.2144568271,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":44.1448054466},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":873053.8373279562,"tput_per_gpu":1196.0837571393,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":48.2524444438},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":311448.3213751095,"tput_per_gpu":426.6842002839,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":50.3579594834},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1103492.1542856935","tput_per_gpu":1511.7842513714,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":194.821211892},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2506588.6238496355","tput_per_gpu":3434.026414674,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":53.2946584656},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":384,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":19579697.57309942,"tput_per_gpu":33481.28285,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":106.0759,"input_tput_per_gpu":16795.64,"kv_cache_precision":"FP8","total_input_tokens":3563226,"output_tput_per_gpu":16685.64,"total_output_tokens":3539890,"tput_requests_per_sec":36.20049,"p50_e2el":11.18067,"p50_tpot":0.010841,"p50_ttft":0.089843,"p50_intvty":92.23836},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"222955.84744357664","tput_per_gpu":305.4495109977,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":117.5282958874},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":512,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2798792.1392275915","tput_per_gpu":3834.3452307418,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":16.7031945215},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"14882.908635693431","tput_per_gpu":20.3895848309,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":186.0755805282},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1318603.9427931388","tput_per_gpu":1806.4874016266,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":35.0400025015},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":16,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":555807.7864120438,"tput_per_gpu":761.4566673845,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":46.3318678023},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":2,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"181939.76357978102","tput_per_gpu":249.2574761043,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":111.2794521935},{"hw":"H100","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1727017.3984578103","tput_per_gpu":2366.0138358872,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":29.0966840321},{"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-04","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1581800.8336760527","tput_per_gpu":3005.4215839845,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":97.2725334284},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":28,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"20627.358490566035","tput_per_gpu":43.73,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":226.7574},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"116386.58231057895","tput_per_gpu":221.1345063901,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":111.1648978301,"output_tput_per_gpu":109.96960856,"p50_itl":"1.8112025520530999","p90_itl":"2.0116438803962002","mean_itl":1.7548068331175,"p50_e2el":"7.1598325664527005","p50_tpot":0.0080534424274,"p50_ttft":0.1486608764972,"p90_e2el":"11.367660816130199","p90_tpot":"0.010400518822700001","p90_ttft":"0.26856373266780004","mean_e2el":8.010735235607699,"mean_tpot":0.0084025974489,"mean_ttft":"0.33345573021220004","p50_intvty":124.1705033617,"p90_intvty":96.1490495857,"mean_intvty":119.0108185094},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":24,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"132901.12090759433","tput_per_gpu":281.7503763241,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":2400.1872437431,"output_tput_per_gpu":149.3480721105,"p50_itl":"1.0697207389749002","mean_itl":0.9500149552184001,"p50_e2el":4.5032773949788,"p50_tpot":"0.0045669084706999995","p50_ttft":"0.38739624302250003","mean_e2el":4.3980563303844,"mean_tpot":0.0043559704721,"mean_ttft":0.4040352574836,"p50_intvty":218.9665079595,"mean_intvty":229.5699675663},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":1077815.880521752,"tput_per_gpu":1476.6077563148,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":7.4419305674},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"434683.60491394164","tput_per_gpu":595.5165387321,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":134.8498766218},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"2022712.1311768615","tput_per_gpu":2771.1156197123,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":44.350265178},{"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-04","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":315972.176627,"tput_per_gpu":600.3471355913,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":154.6468069334},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1029105.5587245986","tput_per_gpu":1409.8746154527,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":90.0582596635},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":715108.9715643796,"tput_per_gpu":979.6992910432,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":118.5059705797},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":28757.24417109489,"tput_per_gpu":39.3974245144,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":81.3907186517},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":1024,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":124231.1320754717,"tput_per_gpu":263.37,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":26.7666},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-06-12","num_gpus":64,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":3077189.621269789,"tput_per_gpu":5846.6602804126,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":2924.443374893,"output_tput_per_gpu":2922.2169055197,"p50_itl":7.6552827929845,"p90_itl":7.7116442720173,"mean_itl":6.711062567133,"p50_e2el":38.6679582684883,"p50_tpot":"0.032407764371699996","p50_ttft":8.710653852496801,"p90_e2el":"46.290526642813305","p90_tpot":"0.038693496149300004","p90_ttft":10.9161524203606,"mean_e2el":"38.945492456456606","mean_tpot":0.0321642369276,"mean_ttft":9.3460364822648,"p50_intvty":30.8568029726,"p90_intvty":25.8441365996,"mean_intvty":31.0904313462},{"ep":4,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-12","num_gpus":16,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"1309788.6061921578","tput_per_gpu":2488.5983517651,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":1245.9821013593,"output_tput_per_gpu":1242.6162504057,"p50_itl":5.3474207799882,"p90_itl":"5.7404845469515005","mean_itl":4.8487014851366,"p50_e2el":22.3651209900272,"p50_tpot":"0.021960609297600003","p50_ttft":1.9167421425227,"p90_e2el":"27.694980081997297","p90_tpot":0.0276786263319,"p90_ttft":3.4288185034529,"mean_e2el":22.7508038702928,"mean_tpot":"0.022517931201300002","mean_ttft":2.0680617058561,"p50_intvty":45.5360771847,"p90_intvty":36.1289605925,"mean_intvty":44.4090529925},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":290009.4731864737,"tput_per_gpu":551.0179990543,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":276.8985417161,"output_tput_per_gpu":274.1194573382,"p50_itl":"3.0831930610583997","p90_itl":"3.3579006591811997","mean_itl":2.8870980598676,"p50_e2el":"12.941892520990201","p50_tpot":0.014303560332,"p50_ttft":0.1666490869829,"p90_e2el":15.4113888972322,"p90_tpot":0.0160534144347,"p90_ttft":0.4035498623969,"mean_e2el":"12.985334434996899","mean_tpot":0.013895969137,"mean_ttft":0.2640588124385,"p50_intvty":69.9126634762,"p90_intvty":62.2920441049,"mean_intvty":71.9633146953},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":2253,"date":"2026-02-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8552620.647857737,"tput_per_gpu":18131.5557734584,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":20,"input_tput_per_gpu":22561.5324552162,"output_tput_per_gpu":7056.6140690639,"p50_itl":6.2325701049995,"mean_itl":"5.4753600121993005","p50_e2el":34.8104832569952,"p50_tpot":0.033561001648,"p50_ttft":"3.3582652070036003","mean_e2el":35.6466374686204,"mean_tpot":0.0327031351828,"mean_ttft":"5.5173329805633005","p50_intvty":29.7964885104,"mean_intvty":30.5781080135},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1091072.7456944527","tput_per_gpu":1494.7696616014,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":96.6651069752},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":128,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":2594780.18101365,"tput_per_gpu":3554.8488479887,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":13.913272014},{"hw":"H200","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-06-25","num_gpus":2,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1815499.6056368614","tput_per_gpu":2487.2344597225,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":49.9618900695},{"hw":"H200","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-25","num_gpus":2,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":1643291.762522044,"tput_per_gpu":2251.3097146552,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":73.4533475827},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2900495.9751145258","tput_per_gpu":3973.6794859069,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":30.3352731133},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":15999.78503832117,"tput_per_gpu":21.9197055025,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":91.8326038197},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":512,"date":"2026-06-12","num_gpus":64,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"2416026.2526028943","tput_per_gpu":4590.4498799455,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":2294.9354675444,"output_tput_per_gpu":2295.5144124011,"p50_itl":"5.6500886534923005","p90_itl":6.1489046631265,"mean_itl":5.0090752380748,"p50_e2el":"24.456593876588197","p50_tpot":"0.023896236045799997","p50_ttft":2.0368058930035,"p90_e2el":30.1070446420345,"p90_tpot":0.0287849468143,"p90_ttft":3.9747144138441,"mean_e2el":24.6713575412567,"mean_tpot":0.0239293030678,"mean_ttft":"2.6372965599966003","p50_intvty":41.8475946622,"p90_intvty":34.7403803263,"mean_intvty":41.7897670135},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":68301.16080873685,"tput_per_gpu":129.7722055366,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":64.322758153,"output_tput_per_gpu":65.4494473836,"p50_itl":1.4941808070289,"p90_itl":1.5961899073096,"mean_itl":1.3736196521622,"p50_e2el":6.8186654859455,"p50_tpot":0.0068330245258,"p50_ttft":"0.14026347204339998","p90_e2el":8.541498763836,"p90_tpot":0.0076474805077,"p90_ttft":1.7545537921367,"mean_e2el":6.9462655912444,"mean_tpot":0.0067458031629,"mean_ttft":0.6276153707469,"p50_intvty":146.3480770819,"p90_intvty":130.7620201186,"mean_intvty":148.2403171049},{"ep":4,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-12","num_gpus":16,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":763179.8276806315,"tput_per_gpu":1450.0416725932,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":725.5459800499,"output_tput_per_gpu":724.4956925433,"p50_itl":4.544449744978899,"p90_itl":4.8183151840349,"mean_itl":4.0822364333777,"p50_e2el":19.175553192501,"p50_tpot":0.0185536376913,"p50_ttft":"1.7517610514769002","p90_e2el":"23.907709729473602","p90_tpot":0.0231280117412,"p90_ttft":3.8986581749865,"mean_e2el":19.5554179629593,"mean_tpot":0.0189764869763,"mean_ttft":"2.1038573255319997","p50_intvty":53.8977863336,"p90_intvty":43.2376120867,"mean_intvty":52.6967926809},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":3226,"date":"2026-02-03","num_gpus":6,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8357326.118558963,"tput_per_gpu":17717.531371345,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":26581.8335725709,"output_tput_per_gpu":13285.3802707321,"p50_itl":9.007933673507099,"mean_itl":7.8290549755462,"p50_e2el":53.5680546145013,"p50_tpot":0.0498719536243,"p50_ttft":5.9892110720102,"mean_e2el":54.2908676289247,"mean_tpot":"0.047827870251299996","mean_ttft":"10.276338087891599","p50_intvty":20.0513500541,"mean_intvty":20.9083112994},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"223022.09073474456","tput_per_gpu":305.5402643066,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":147.5860442873},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":2342436.214838759,"tput_per_gpu":3209.1376143291,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":24.890937309},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":84484.40501226278,"tput_per_gpu":115.7436348668,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":59.865733731},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"161109.65281304094","tput_per_gpu":275.4975063103,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":318.5231782776},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":512,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":4649408.073352456,"tput_per_gpu":7950.4878054327,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":67.6876925503},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":32,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"175527.12739160584","tput_per_gpu":240.4721645265,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":31.0766654313},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":5,"date":"2026-02-03","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"148704.27171976416","tput_per_gpu":315.2530560459,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":4758.1616900336,"output_tput_per_gpu":37.5712664216,"p50_itl":0.7895317499642001,"mean_itl":0.6970344406914001,"p50_e2el":3.763185795513,"p50_tpot":"0.0032010823524999997","p50_ttft":0.8078803464887,"mean_e2el":3.6912806261634,"mean_tpot":"0.0030393795248000004","mean_ttft":0.8892023601802,"p50_intvty":312.394337251,"mean_intvty":329.0145214977},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":12,"date":"2026-02-03","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"342794.63497910375","tput_per_gpu":726.7246261557,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":10986.2117180919,"output_tput_per_gpu":85.5066829097,"p50_itl":0.9008071229910001,"mean_itl":0.811457401638,"p50_e2el":3.9414252254937,"p50_tpot":0.0038037855643,"p50_ttft":0.5019430844986,"mean_e2el":3.8309279630329,"mean_tpot":"0.0035232093452000003","mean_ttft":0.6121470498692,"p50_intvty":262.8959974444,"mean_intvty":283.832126342},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":181,"date":"2026-01-28","num_gpus":14,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"4140818.3737821053","tput_per_gpu":7867.554910186,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":16321.3943077224,"output_tput_per_gpu":1527.1753620337,"p50_itl":2.5650058488827,"mean_itl":"2.2340110614118998","p50_e2el":"13.253701606416099","p50_tpot":0.0105083989532,"p50_ttft":3.7521123094484,"mean_e2el":12.9124770842217,"mean_tpot":0.0097231959364,"mean_ttft":3.9702618364799,"p50_intvty":95.1619751456,"mean_intvty":102.8468423899},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1536,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":37095611.1754386,"tput_per_gpu":63433.49511,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":448.1003,"input_tput_per_gpu":31826.71,"kv_cache_precision":"FP8","total_input_tokens":14261558,"output_tput_per_gpu":31606.78,"total_output_tokens":14163009,"tput_requests_per_sec":34.27804,"p50_e2el":46.54151,"p50_tpot":0.041247,"p50_ttft":4.345390999999999,"p50_intvty":24.24393},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"124798.62561729927","tput_per_gpu":170.9741170957,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":43.24832588},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"240134.92540109492","tput_per_gpu":328.9848477995,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":22.2767288233},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":32,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"175738.16320846716","tput_per_gpu":240.7612835956,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":30.7241214251},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":77021.12361292397,"tput_per_gpu":131.7061213781,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":195.3116814114},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":691913.0007065497,"tput_per_gpu":1183.1712312082,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":164.7650461411},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":3853229.415925526,"tput_per_gpu":7321.1358902585,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":58.8994463717},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":"380489.06640255474","tput_per_gpu":521.2700209715,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":260.6937544424,"output_tput_per_gpu":260.576266529,"p50_itl":0.0256808307022,"p90_itl":"0.027027357369699998","mean_itl":0.0296527988001,"p50_e2el":27.8668094421737,"p50_tpot":0.0300904406837,"p50_ttft":"0.14745885320010002","p90_e2el":"30.800893590040502","p90_tpot":"0.031032537995599997","p90_ttft":"0.34905903283510004","mean_e2el":27.6633499625692,"mean_tpot":"0.029632768275499998","mean_ttft":0.3586359859823,"p50_intvty":33.2331457193,"p90_intvty":32.224241541,"mean_intvty":33.7464252649},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":"121122.02125379562","tput_per_gpu":165.9371691177,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":82.6610299994,"output_tput_per_gpu":83.2761391183,"p50_itl":0.0109876892529,"p90_itl":"0.011365810409199999","mean_itl":0.0116111882162,"p50_e2el":11.0147727825679,"p50_tpot":"0.011661375634600001","p50_ttft":0.1333247688599,"p90_e2el":11.9296679095365,"p90_tpot":0.0118361043107,"p90_ttft":"0.26197249153630003","mean_e2el":"10.943786948826201","mean_tpot":0.0116068394936,"mean_ttft":0.1677336126217,"p50_intvty":85.7531762403,"p90_intvty":84.4872581173,"mean_intvty":86.1560979244},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":183266.0532170803,"tput_per_gpu":251.0744929074,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":126.2019020216,"output_tput_per_gpu":124.8725908857,"p50_itl":0.0144350668415,"p90_itl":"0.014863688545300001","mean_itl":"0.015474647323199999","p50_e2el":14.3000503134681,"p50_tpot":"0.015595146213999999","p50_ttft":0.1161580585176,"p90_e2el":"15.855537527659902","p90_tpot":0.0158545911467,"p90_ttft":0.1894970354158,"mean_e2el":"14.333988135671799","mean_tpot":0.0154678811459,"mean_ttft":"0.17633013413430002","p50_intvty":64.12251519,"p90_intvty":63.0732127211,"mean_intvty":64.6500959356},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":74410.64327175183,"tput_per_gpu":101.9425812823,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":51.2193177138,"output_tput_per_gpu":50.7232635685,"p50_itl":0.0091467984021,"p90_itl":0.0093886662275,"mean_itl":0.009428594781,"p50_e2el":8.8651502002031,"p50_tpot":0.0094564636767,"p50_ttft":0.1321904123761,"p90_e2el":9.6874399858527,"p90_tpot":0.009554688798000002,"p90_ttft":"0.16278559956699998","mean_e2el":8.7885827974416,"mean_tpot":0.009426644845,"mean_ttft":0.145154412929,"p50_intvty":105.7477757213,"p90_intvty":104.6606562638,"mean_intvty":106.0822823433},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":32,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":"258321.67715262776","tput_per_gpu":353.9006976991,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":176.6700353834,"output_tput_per_gpu":177.2306623157,"p50_itl":0.0194923467934,"p90_itl":0.0203348910436,"mean_itl":0.021793789255,"p50_e2el":20.3711569327861,"p50_tpot":0.0219070033783,"p50_ttft":"0.13636806095020002","p90_e2el":22.4463372718543,"p90_tpot":0.0227651224824,"p90_ttft":0.3565663445741,"mean_e2el":20.3697125215782,"mean_tpot":0.021787442159,"mean_ttft":0.2607876803464,"p50_intvty":45.647502889,"p90_intvty":43.9268447062,"mean_intvty":45.8979990721},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-15","num_gpus":28,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4505281.678175189,"tput_per_gpu":9551.1971577314,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":11149.8462688336,"output_tput_per_gpu":8352.2103244046,"p50_itl":5.1700048234779,"mean_itl":4.597570581597299,"p50_e2el":27.3020250815316,"p50_tpot":0.0270044123665,"p50_ttft":"1.6967013999820002","mean_e2el":28.4900546606903,"mean_tpot":0.0268333518166,"mean_ttft":3.7952963321803,"p50_intvty":37.030985397,"mean_intvty":37.2670550751},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":167,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"374888.60136056604","tput_per_gpu":794.7638348844,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":3582.6101511316,"output_tput_per_gpu":446.2830453535,"p50_itl":2.2360554800835,"mean_itl":1.9413702742693,"p50_e2el":9.700842658989101,"p50_tpot":0.007865151654,"p50_ttft":2.297005918459,"mean_e2el":10.0203021533914,"mean_tpot":0.008250721397599999,"mean_ttft":"2.4443082417097997","p50_intvty":127.1431300996,"mean_intvty":121.2015230925},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":2253,"date":"2026-06-15","num_gpus":44,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":2727633.903797736,"tput_per_gpu":5782.5838760512,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":10600.9400864138,"output_tput_per_gpu":3975.7002971652,"p50_itl":3.4569424494985,"mean_itl":3.0124620333579,"p50_e2el":13.9837220779737,"p50_tpot":"0.012311166758100001","p50_ttft":1.4789765500464,"mean_e2el":15.4243435602334,"mean_tpot":"0.012845079540500001","mean_ttft":3.5924411067637,"p50_intvty":81.2270696719,"mean_intvty":77.8508219312},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":90,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"216139.24203301888","tput_per_gpu":458.21519311,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":2063.8089833224,"output_tput_per_gpu":257.5159693334,"p50_itl":2.0467672490049,"mean_itl":"1.7847762235442999","p50_e2el":9.3425197189208,"p50_tpot":0.0072544571491,"p50_ttft":"2.7076359684579003","mean_e2el":9.4964623417392,"mean_tpot":0.0072864619669,"mean_ttft":2.7896700398471,"p50_intvty":137.8462894538,"mean_intvty":137.2408179084},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2002916.6404117518","tput_per_gpu":2743.9957973641,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":85.0708523227},{"ep":8,"hw":"B300","tp":8,"conc":768,"date":"2026-10-20","num_gpus":12,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"212713836.90526316","tput_per_gpu":404156.29012,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":121011,"num_prefill_gpu":4,"duration_seconds":3629.46428,"input_tput_per_gpu":401301.32568,"total_input_tokens":17481614885,"output_tput_per_gpu":2854.96445,"total_output_tokens":124520934,"avg_output_tokens_expected":1367.72444,"theoretical_cache_hit_rate":0.97251,"stdev_output_tokens_expected":2521.49685,"avg_isl":144434.02623,"avg_osl":1027.54211,"p50_itl":0.005,"p75_isl":186537.5,"p75_osl":1024.0,"p75_qps":39.0,"p90_isl":316802.0,"p90_itl":0.00522,"p90_osl":2392.0,"p90_qps":44.0,"p95_isl":445854.0,"p95_osl":3962.5,"p95_qps":48.0,"std_qps":8.85983,"mean_itl":"0.0049900000000000005","mean_qps":33.34527,"p50_e2el":"3.6631799999999997","p50_tpot":0.005,"p50_ttft":1.0478,"p90_e2el":13.70923,"p90_tpot":0.00522,"p90_ttft":2.97306,"mean_e2el":6.5215,"mean_tpot":0.00497,"mean_ttft":1.52207,"stdev_isl":141632.84183,"stdev_osl":1992.48633,"median_qps":33.0,"p50_intvty":200.0,"p90_intvty":191.5708812261,"mean_intvty":200.4008016032},{"ep":8,"hw":"B300","tp":8,"conc":256,"date":"2026-10-20","num_gpus":12,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":79356464.4,"tput_per_gpu":150777.28236,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":43250,"num_prefill_gpu":4,"duration_seconds":3626.78334,"input_tput_per_gpu":149767.04632,"total_input_tokens":6519045779,"output_tput_per_gpu":1010.23604,"total_output_tokens":43987430,"avg_output_tokens_expected":1340.70545,"theoretical_cache_hit_rate":0.97397,"stdev_output_tokens_expected":2414.91268,"avg_isl":150706.85651,"avg_osl":1016.57542,"p50_itl":0.0039,"p75_isl":197123.0,"p75_osl":1016.0,"p75_qps":15.0,"p90_isl":329663.5,"p90_itl":"0.0041600000000000005","p90_osl":2372.0,"p90_qps":17.0,"p95_isl":459656.6,"p95_osl":3898.55,"p95_qps":19.0,"std_qps":4.27027,"mean_itl":0.00395,"mean_qps":11.92719,"p50_e2el":"2.1243000000000003","p50_tpot":0.0039,"p50_ttft":0.29189,"p90_e2el":9.65976,"p90_tpot":"0.0041600000000000005","p90_ttft":0.8287899999999999,"mean_e2el":4.38199,"mean_tpot":0.00393,"mean_ttft":0.46184,"stdev_isl":146059.8395,"stdev_osl":2032.85491,"median_qps":12.0,"p50_intvty":256.4102564103,"p90_intvty":240.3846153846,"mean_intvty":253.164556962},{"ep":1,"hw":"B300","tp":2,"conc":36,"date":"2026-10-14","num_gpus":2,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":28621538.12631579,"tput_per_gpu":54380.92244,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791961200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38390415472/attempts/1","total_requests":3670,"duration_seconds":3616.89904,"input_tput_per_gpu":53889.85634,"total_input_tokens":390290594,"output_tput_per_gpu":491.0661,"total_output_tokens":3552275,"avg_output_tokens_expected":1350.55914,"theoretical_cache_hit_rate":0.96439,"stdev_output_tokens_expected":2089.06783,"avg_isl":106220.25586,"avg_osl":967.9218,"p50_itl":0.00349,"p75_isl":151118.0,"p75_osl":983.5,"p75_qps":2.0,"p90_isl":203872.1,"p90_itl":"0.0048200000000000005","p90_osl":2323.1,"p90_qps":2.0,"p95_isl":229259.2,"p95_osl":3713.3,"p95_qps":3.0,"std_qps":0.97608,"mean_itl":"0.0044599999999999996","mean_qps":1.0186,"p50_e2el":25.20184,"p50_tpot":0.00349,"p50_ttft":22.18727,"p90_e2el":"42.950540000000004","p90_tpot":"0.0048200000000000005","p90_ttft":37.53235,"mean_e2el":26.53548,"mean_tpot":0.00447,"mean_ttft":"22.850669999999997","stdev_isl":64210.87885,"stdev_osl":2292.67972,"median_qps":1.0,"p50_intvty":286.5329512894,"p90_intvty":207.468879668,"mean_intvty":224.2152466368},{"ep":1,"hw":"B300","tp":8,"conc":1,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"1065963.1368421053","tput_per_gpu":2025.32996,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31893747354/attempts/2","total_requests":239,"duration_seconds":3585.92273,"input_tput_per_gpu":2010.1993,"total_input_tokens":61747004,"output_tput_per_gpu":15.13066,"total_output_tokens":441439,"avg_output_tokens_expected":2032.20502,"theoretical_cache_hit_rate":0.97712,"stdev_output_tokens_expected":2822.64994,"avg_isl":241286.00418,"avg_osl":1816.14644,"p50_itl":0.00423,"p75_isl":375084.0,"p75_osl":2533.5,"p75_qps":0.0,"p90_isl":414776.0,"p90_itl":"0.0049299999999999995","p90_osl":4819.0,"p90_qps":0.0,"p95_isl":443936.6,"p95_osl":6426.9,"p95_qps":1.0,"std_qps":0.25253,"mean_itl":0.00454,"mean_qps":0.06652,"p50_e2el":5.26484,"p50_tpot":0.00423,"p50_ttft":1.37168,"p90_e2el":23.00267,"p90_tpot":"0.0049299999999999995","p90_ttft":"3.1356599999999997","mean_e2el":9.62951,"mean_tpot":0.00454,"mean_ttft":1.69433,"stdev_isl":141387.34077,"stdev_osl":2495.40571,"median_qps":0.0,"p50_intvty":236.4066193853,"p90_intvty":202.8397565923,"mean_intvty":220.2643171806},{"ep":1,"hw":"B300","tp":8,"conc":8,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"2726546.6263157893","tput_per_gpu":5180.43859,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31893747354/attempts/2","total_requests":1073,"duration_seconds":3607.96498,"input_tput_per_gpu":5139.80599,"total_input_tokens":166565454,"output_tput_per_gpu":40.6326,"total_output_tokens":1172895,"avg_output_tokens_expected":2037.65977,"theoretical_cache_hit_rate":0.97541,"stdev_output_tokens_expected":2704.81495,"avg_isl":138260.87605,"avg_osl":1093.01771,"p50_itl":0.00782,"p75_isl":150994.0,"p75_osl":1029.0,"p75_qps":0.0,"p90_isl":290547.4,"p90_itl":0.01565,"p90_osl":2653.0,"p90_qps":1.0,"p95_isl":384176.8,"p95_osl":4579.8,"p95_qps":1.0,"std_qps":0.57713,"mean_itl":0.009470000000000001,"mean_qps":0.2972,"p50_e2el":5.41529,"p50_tpot":0.00782,"p50_ttft":1.04294,"p90_e2el":"22.839419999999997","p90_tpot":0.01565,"p90_ttft":2.27809,"mean_e2el":10.22597,"mean_tpot":0.009470000000000001,"mean_ttft":1.31593,"stdev_isl":113606.32183,"stdev_osl":1964.7966,"median_qps":0.0,"p50_intvty":127.8772378517,"p90_intvty":63.8977635783,"mean_intvty":105.5966209081},{"ep":1,"hw":"B300","tp":8,"conc":16,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":4026683.010526316,"tput_per_gpu":7650.69772,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31893747354/attempts/2","total_requests":1741,"duration_seconds":3624.88897,"input_tput_per_gpu":7592.28324,"total_input_tokens":259528291,"output_tput_per_gpu":58.41448,"total_output_tokens":1715776,"avg_output_tokens_expected":1722.86491,"theoretical_cache_hit_rate":0.96967,"stdev_output_tokens_expected":2728.82211,"avg_isl":126461.49914,"avg_osl":972.98564,"p50_itl":0.016,"p75_isl":138914.0,"p75_osl":812.0,"p75_qps":1.0,"p90_isl":202028.0,"p90_itl":"0.031129999999999998","p90_osl":2372.0,"p90_qps":1.0,"p95_isl":375478.0,"p95_osl":4561.0,"p95_qps":2.0,"std_qps":0.78031,"mean_itl":0.01877,"mean_qps":0.48013,"p50_e2el":7.24245,"p50_tpot":0.016,"p50_ttft":1.11374,"p90_e2el":38.33878,"p90_tpot":"0.031129999999999998","p90_ttft":3.22504,"mean_e2el":17.70412,"mean_tpot":0.01877,"mean_ttft":1.67245,"stdev_isl":123628.52302,"stdev_osl":2076.4305,"median_qps":0.0,"p50_intvty":62.5,"p90_intvty":32.1233536781,"mean_intvty":53.2765050613},{"ep":1,"hw":"B300","tp":8,"conc":4,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":1642046.4,"tput_per_gpu":3119.88816,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/31893747354/attempts/2","total_requests":534,"duration_seconds":3604.14142,"input_tput_per_gpu":3096.44211,"total_input_tokens":99809606,"output_tput_per_gpu":23.44605,"total_output_tokens":676067,"avg_output_tokens_expected":1904.40824,"theoretical_cache_hit_rate":0.9748,"stdev_output_tokens_expected":2731.82341,"avg_isl":167191.2397,"avg_osl":1265.96067,"p50_itl":0.00504,"p75_isl":204130.0,"p75_osl":1356.75,"p75_qps":0.0,"p90_isl":505988.6,"p90_itl":0.00826,"p90_osl":3838.8,"p90_qps":1.0,"p95_isl":561780.45,"p95_osl":5379.3,"p95_qps":1.0,"std_qps":0.38264,"mean_itl":0.00625,"mean_qps":0.14839,"p50_e2el":3.15423,"p50_tpot":0.00504,"p50_ttft":0.95808,"p90_e2el":21.81182,"p90_tpot":0.00826,"p90_ttft":2.30494,"mean_e2el":8.05834,"mean_tpot":0.00625,"mean_ttft":1.24218,"stdev_isl":173726.50231,"stdev_osl":2260.74194,"median_qps":0.0,"p50_intvty":198.4126984127,"p90_intvty":121.0653753027,"mean_intvty":160.0},{"ep":1,"hw":"B300","tp":4,"conc":1,"date":"2026-10-20","num_gpus":4,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":3950562.231578947,"tput_per_gpu":7506.06824,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"serving_config":"agg","total_requests":329,"duration_seconds":3593.00457,"input_tput_per_gpu":7460.54206,"total_input_tokens":107888015,"output_tput_per_gpu":45.52617,"total_output_tokens":654305,"avg_output_tokens_expected":2142.1459,"theoretical_cache_hit_rate":0.98476,"stdev_output_tokens_expected":2724.21336,"avg_isl":325905.91793,"avg_osl":1988.76292,"p50_itl":0.00158,"p75_isl":488305.0,"p75_osl":2682.0,"p75_qps":0.0,"p90_isl":593544.8,"p90_itl":0.00161,"p90_osl":5075.6,"p90_qps":0.0,"p95_isl":637954.8,"p95_osl":6526.4,"p95_qps":1.0,"std_qps":0.29391,"mean_itl":0.00158,"mean_qps":0.09139,"p50_e2el":"1.8729200000000001","p50_tpot":0.00158,"p50_ttft":0.13891,"p90_e2el":9.08038,"p90_tpot":0.00161,"p90_ttft":0.7808200000000001,"mean_e2el":"3.6280799999999997","mean_tpot":0.00157,"mean_ttft":0.5194099999999999,"stdev_isl":192275.3097,"stdev_osl":2494.18229,"median_qps":0.0,"p50_intvty":632.9113924051,"p90_intvty":621.1180124224,"mean_intvty":632.9113924051},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2103176.6817496354","tput_per_gpu":2881.352053997,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":2559.1367265103,"output_tput_per_gpu":322.2153274867,"p50_itl":0.019108154811,"p90_itl":0.019779410027,"mean_itl":0.023619761254,"p50_e2el":22.3647715924308,"p50_tpot":0.0239226536836,"p50_ttft":0.2713311072439,"p90_e2el":"25.429598224349302","p90_tpot":0.0249109853082,"p90_ttft":0.6585218762979,"mean_e2el":"22.472380074547303","mean_tpot":"0.023591871304699998","mean_ttft":0.6371657464799,"p50_intvty":41.8013826236,"p90_intvty":40.1429324303,"mean_intvty":42.3874811406},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":24,"date":"2026-02-03","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"471076.92697122646","tput_per_gpu":998.683085179,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":15090.997197889,"output_tput_per_gpu":117.9134531347,"p50_itl":1.01786820899,"mean_itl":0.8924206767121,"p50_e2el":5.4529116555059,"p50_tpot":0.0042212370276,"p50_ttft":1.9026468619995,"mean_e2el":5.5736468521415,"mean_tpot":0.0039263351987,"mean_ttft":"1.9704985239166999","p50_intvty":236.8973818482,"mean_intvty":254.6904299777},{"ep":8,"hw":"B300","tp":8,"conc":2048,"date":"2026-10-20","num_gpus":16,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":364384676.2736842,"tput_per_gpu":692330.88492,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":284688,"num_prefill_gpu":8,"duration_seconds":3628.75044,"input_tput_per_gpu":687340.20199,"total_input_tokens":39924417912,"output_tput_per_gpu":4990.68293,"total_output_tokens":290534488,"avg_output_tokens_expected":1357.92731,"theoretical_cache_hit_rate":0.97132,"stdev_output_tokens_expected":2534.09449,"avg_isl":140177.93841,"avg_osl":1017.81278,"p50_itl":0.00855,"p75_isl":178513.75,"p75_osl":1012.0,"p75_qps":88.0,"p90_isl":299907.3,"p90_itl":0.00945,"p90_osl":2362.3,"p90_qps":96.0,"p95_isl":430272.0,"p95_osl":3901.0,"p95_qps":103.0,"std_qps":16.00826,"mean_itl":0.00848,"mean_qps":78.46885,"p50_e2el":5.888439999999999,"p50_tpot":0.008539999999999999,"p50_ttft":1.47574,"p90_e2el":22.63406,"p90_tpot":0.009439999999999999,"p90_ttft":4.48924,"mean_e2el":10.72106,"mean_tpot":0.00845,"mean_ttft":2.27821,"stdev_isl":137799.50797,"stdev_osl":1992.80801,"median_qps":79.0,"p50_intvty":116.9590643275,"p90_intvty":105.8201058201,"mean_intvty":117.9245283019},{"ep":8,"hw":"B300","tp":8,"conc":1280,"date":"2026-10-20","num_gpus":14,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"221960558.27894738","tput_per_gpu":421725.06073,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":152422,"num_prefill_gpu":6,"duration_seconds":3629.25303,"input_tput_per_gpu":418674.78719,"total_input_tokens":21277138253,"output_tput_per_gpu":3050.27354,"total_output_tokens":155167307,"avg_output_tokens_expected":1337.53642,"theoretical_cache_hit_rate":0.97115,"stdev_output_tokens_expected":2456.08687,"avg_isl":139564.33037,"avg_osl":1016.80206,"p50_itl":0.00522,"p75_isl":176767.75,"p75_osl":1013.0,"p75_qps":48.0,"p90_isl":299968.8,"p90_itl":0.00562,"p90_osl":2363.0,"p90_qps":54.0,"p95_isl":431015.55,"p95_osl":3911.0,"p95_qps":58.0,"std_qps":10.13321,"mean_itl":0.00528,"mean_qps":42.00083,"p50_e2el":"4.2142100000000005","p50_tpot":0.00522,"p50_ttft":1.31843,"p90_e2el":15.6727,"p90_tpot":0.00562,"p90_ttft":4.13138,"mean_e2el":12.4305,"mean_tpot":0.00526,"mean_ttft":7.18499,"stdev_isl":136901.96693,"stdev_osl":1989.02938,"median_qps":42.0,"p50_intvty":191.5708812261,"p90_intvty":177.9359430605,"mean_intvty":189.3939393939},{"ep":8,"hw":"B300","tp":8,"conc":3584,"date":"2026-10-20","num_gpus":18,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":418295556.4473684,"tput_per_gpu":794761.55725,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":384991,"num_prefill_gpu":10,"duration_seconds":3629.92898,"input_tput_per_gpu":788812.24848,"total_input_tokens":51621515233,"output_tput_per_gpu":5949.30877,"total_output_tokens":391121535,"avg_output_tokens_expected":1361.80017,"theoretical_cache_hit_rate":0.969,"stdev_output_tokens_expected":2580.18717,"avg_isl":133873.21772,"avg_osl":1009.68654,"p50_itl":0.01182,"p75_isl":166540.0,"p75_osl":1012.0,"p75_qps":117.0,"p90_isl":285756.0,"p90_itl":0.01233,"p90_osl":2339.0,"p90_qps":124.0,"p95_isl":424609.0,"p95_osl":3855.0,"p95_qps":128.0,"std_qps":17.59557,"mean_itl":0.01165,"mean_qps":106.08267,"p50_e2el":15.64156,"p50_tpot":0.01182,"p50_ttft":9.252799999999999,"p90_e2el":38.42618,"p90_tpot":0.01233,"p90_ttft":15.56842,"mean_e2el":21.42868,"mean_tpot":0.0116,"mean_ttft":9.85403,"stdev_isl":135679.72414,"stdev_osl":1970.89101,"median_qps":108.0,"p50_intvty":84.6023688663,"p90_intvty":81.103000811,"mean_intvty":85.8369098712},{"ep":1,"hw":"B300","tp":2,"conc":40,"date":"2026-10-14","num_gpus":2,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"19540894.968421053","tput_per_gpu":37127.70044,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791961200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38390415472/attempts/1","total_requests":2557,"duration_seconds":3629.28313,"input_tput_per_gpu":36797.4585,"total_input_tokens":268968594,"output_tput_per_gpu":330.24194,"total_output_tokens":2434643,"avg_output_tokens_expected":1374.46045,"theoretical_cache_hit_rate":0.96138,"stdev_output_tokens_expected":2087.86814,"avg_isl":104457.09464,"avg_osl":937.45913,"p50_itl":0.00257,"p75_isl":145778.0,"p75_osl":994.0,"p75_qps":1.0,"p90_isl":199477.2,"p90_itl":0.00317,"p90_osl":2214.8,"p90_qps":2.0,"p95_isl":228889.0,"p95_osl":3718.8,"p95_qps":2.0,"std_qps":0.8377,"mean_itl":"0.0030099999999999997","mean_qps":0.70705,"p50_e2el":48.65643,"p50_tpot":0.00257,"p50_ttft":"45.991730000000004","p90_e2el":74.44484,"p90_tpot":0.00317,"p90_ttft":71.92175,"mean_e2el":"51.137449999999994","mean_tpot":"0.0030099999999999997","mean_ttft":"48.572480000000006","stdev_isl":63167.60019,"stdev_osl":1745.0379,"median_qps":0.0,"p50_intvty":389.1050583658,"p90_intvty":315.4574132492,"mean_intvty":332.2259136213},{"ep":1,"hw":"B300","tp":2,"conc":44,"date":"2026-10-14","num_gpus":2,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":37091459.02105263,"tput_per_gpu":70473.77214,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791961200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38390415472/attempts/1","total_requests":5011,"duration_seconds":3629.35658,"input_tput_per_gpu":69810.76068,"total_input_tokens":509445576,"output_tput_per_gpu":663.01146,"total_output_tokens":4812625,"avg_output_tokens_expected":1321.11575,"theoretical_cache_hit_rate":0.96058,"stdev_output_tokens_expected":2092.59909,"avg_isl":101124.78288,"avg_osl":960.4091,"p50_itl":"0.0051600000000000005","p75_isl":142223.5,"p75_osl":996.5,"p75_qps":2.0,"p90_isl":204474.0,"p90_itl":0.00732,"p90_osl":2311.0,"p90_qps":3.0,"p95_isl":232606.0,"p95_osl":3731.5,"p95_qps":3.0,"std_qps":1.12649,"mean_itl":0.0057,"mean_qps":1.38666,"p50_e2el":"17.887330000000002","p50_tpot":"0.0051600000000000005","p50_ttft":13.84016,"p90_e2el":31.75757,"p90_tpot":0.00733,"p90_ttft":22.8567,"mean_e2el":19.54411,"mean_tpot":0.00571,"mean_ttft":14.25384,"stdev_isl":65219.11582,"stdev_osl":2108.73235,"median_qps":1.0,"p50_intvty":193.7984496124,"p90_intvty":136.6120218579,"mean_intvty":175.4385964912},{"ep":1,"hw":"B300","tp":4,"conc":4,"date":"2026-10-14","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"3626240.4210526315","tput_per_gpu":6889.8568,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791961200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38390415472/attempts/1","total_requests":1117,"duration_seconds":3605.38096,"input_tput_per_gpu":6817.35309,"total_input_tokens":98366314,"output_tput_per_gpu":72.50371,"total_output_tokens":1045615,"avg_output_tokens_expected":1656.40664,"theoretical_cache_hit_rate":0.95571,"stdev_output_tokens_expected":2494.6192,"avg_isl":88018.46016,"avg_osl":936.09132,"p50_itl":0.00182,"p75_isl":108316.0,"p75_osl":801.0,"p75_qps":1.0,"p90_isl":186808.8,"p90_itl":0.00188,"p90_osl":2469.6,"p90_qps":1.0,"p95_isl":222727.0,"p95_osl":4180.4,"p95_qps":1.0,"std_qps":0.54499,"mean_itl":0.00182,"mean_qps":0.31041,"p50_e2el":"1.9666700000000001","p50_tpot":0.00182,"p50_ttft":0.561,"p90_e2el":9.93391,"p90_tpot":0.00188,"p90_ttft":5.47677,"mean_e2el":3.77691,"mean_tpot":0.00183,"mean_ttft":2.05748,"stdev_isl":59020.72314,"stdev_osl":1839.18148,"median_qps":0.0,"p50_intvty":549.4505494505,"p90_intvty":531.914893617,"mean_intvty":549.4505494505},{"ep":1,"hw":"B300","tp":2,"conc":64,"date":"2026-10-14","num_gpus":2,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":59397867.71052632,"tput_per_gpu":112855.94865,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791961200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38390415472/attempts/1","total_requests":9481,"duration_seconds":3628.41477,"input_tput_per_gpu":111661.35603,"total_input_tokens":812558930,"output_tput_per_gpu":1194.59262,"total_output_tokens":8711239,"avg_output_tokens_expected":1154.40591,"theoretical_cache_hit_rate":0.95215,"stdev_output_tokens_expected":2067.38231,"avg_isl":85466.45143,"avg_osl":914.35028,"p50_itl":0.01642,"p75_isl":123020.0,"p75_osl":1006.0,"p75_qps":4.0,"p90_isl":185585.0,"p90_itl":0.02656,"p90_osl":1997.0,"p90_qps":5.0,"p95_isl":222201.0,"p95_osl":3366.0,"p95_qps":7.0,"std_qps":2.15798,"mean_itl":0.01874,"mean_qps":2.62576,"p50_e2el":8.31082,"p50_tpot":0.01642,"p50_ttft":0.61039,"p90_e2el":37.07739,"p90_tpot":0.02656,"p90_ttft":1.44391,"mean_e2el":16.88517,"mean_tpot":0.01874,"mean_ttft":0.84,"stdev_isl":64354.13104,"stdev_osl":1785.41377,"median_qps":2.0,"p50_intvty":60.9013398295,"p90_intvty":37.6506024096,"mean_intvty":53.3617929562},{"ep":1,"hw":"B300","tp":4,"conc":16,"date":"2026-10-20","num_gpus":4,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"15619879.842105264","tput_per_gpu":29677.7717,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792479600,"serving_config":"agg","total_requests":2737,"duration_seconds":3610.93679,"input_tput_per_gpu":29485.03764,"total_input_tokens":426624712,"output_tput_per_gpu":192.73406,"total_output_tokens":2783804,"avg_output_tokens_expected":1649.8238,"theoretical_cache_hit_rate":0.9763,"stdev_output_tokens_expected":2534.0107,"avg_isl":155598.98758,"avg_osl":1017.09974,"p50_itl":0.00227,"p75_isl":184554.0,"p75_osl":962.0,"p75_qps":1.0,"p90_isl":284947.2,"p90_itl":0.00298,"p90_osl":2555.8,"p90_qps":2.0,"p95_isl":526591.2,"p95_osl":4395.6,"p95_qps":2.0,"std_qps":0.9147,"mean_itl":0.00232,"mean_qps":0.75789,"p50_e2el":0.95196,"p50_tpot":0.00226,"p50_ttft":0.07665999999999999,"p90_e2el":5.52145,"p90_tpot":0.00298,"p90_ttft":0.20107,"mean_e2el":"2.3514899999999996","mean_tpot":0.00231,"mean_ttft":0.11864,"stdev_isl":152407.18374,"stdev_osl":1970.26448,"median_qps":1.0,"p50_intvty":440.5286343612,"p90_intvty":335.5704697987,"mean_intvty":431.0344827586},{"ep":16,"hw":"GB300 NVL72","tp":20,"conc":600,"date":"2026-10-26","num_gpus":28,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"54233975.330188684","dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":114976.0277,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":115583,"duration_seconds":3623.11339,"input_tput_per_gpu":113914.85464,"output_tput_per_gpu":1061.17306,"tput_requests_per_sec":31.91082,"avg_isl":99983.04409,"p50_itl":"0.0030499999999999998","p90_itl":0.00348,"mean_itl":0.00318,"p50_e2el":2.17627,"p50_tpot":"0.0030499999999999998","p50_ttft":0.76302,"p75_ttft":0.9853,"p90_e2el":8.532020000000001,"p90_tpot":0.00348,"p90_ttft":"1.6174600000000001","mean_e2el":"4.0590399999999995","mean_tpot":0.00318,"mean_ttft":1.21022,"stdev_isl":65399.92498,"p50_intvty":327.77878,"p75_intvty":313.11093,"p90_intvty":287.73285,"mean_intvty":314.54732},{"ep":4,"hw":"GB300 NVL72","tp":10,"conc":192,"date":"2026-10-26","num_gpus":12,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":42573355.25471698,"dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":90255.51314,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":37902,"duration_seconds":3608.53058,"input_tput_per_gpu":89445.60949,"output_tput_per_gpu":809.90365,"tput_requests_per_sec":10.50471,"avg_isl":102190.03224,"p50_itl":0.00211,"p90_itl":0.00322,"mean_itl":0.00233,"p50_e2el":"1.4600799999999998","p50_tpot":0.00211,"p50_ttft":0.49893,"p75_ttft":0.7442799999999999,"p90_e2el":5.732489999999999,"p90_tpot":0.00322,"p90_ttft":"1.0500999999999998","mean_e2el":"2.7594499999999997","mean_tpot":0.00233,"mean_ttft":0.69353,"stdev_isl":65735.31321,"p50_intvty":472.9965,"p75_intvty":409.10619,"p90_intvty":310.74721,"mean_intvty":429.30841},{"ep":16,"hw":"GB300 NVL72","tp":20,"conc":700,"date":"2026-10-26","num_gpus":28,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"62174222.858490564","dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":131809.35246,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":134182,"duration_seconds":3627.83502,"input_tput_per_gpu":130583.37899,"output_tput_per_gpu":1225.97346,"tput_requests_per_sec":36.99504,"avg_isl":98855.12777,"p50_itl":0.0033,"p90_itl":0.00373,"mean_itl":"0.0034100000000000003","p50_e2el":"2.3994400000000002","p50_tpot":0.0033,"p50_ttft":0.82904,"p75_ttft":"1.0569300000000001","p90_e2el":10.49072,"p90_tpot":0.00373,"p90_ttft":"2.1156599999999997","mean_e2el":4.6415,"mean_tpot":"0.0034100000000000003","mean_ttft":"1.6003399999999999","stdev_isl":65339.10412,"p50_intvty":302.98935,"p75_intvty":288.61071,"p90_intvty":267.86662,"mean_intvty":293.53215},{"ep":4,"hw":"GB300 NVL72","tp":8,"conc":704,"date":"2026-10-26","num_gpus":20,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":77335578.80660377,"dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":163951.42707,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":120437,"duration_seconds":3629.48488,"input_tput_per_gpu":162411.53815,"output_tput_per_gpu":1539.88892,"tput_requests_per_sec":33.1871,"avg_isl":97888.55959,"p50_itl":0.00701,"p90_itl":0.00743,"mean_itl":0.00698,"p50_e2el":4.248060000000001,"p50_tpot":0.00701,"p50_ttft":0.87248,"p75_ttft":1.37579,"p90_e2el":17.05079,"p90_tpot":0.00743,"p90_ttft":"3.5957600000000003","mean_e2el":7.991149999999999,"mean_tpot":0.00698,"mean_ttft":1.77127,"stdev_isl":64959.04874,"p50_intvty":142.66976,"p75_intvty":138.93634,"p90_intvty":134.63756,"mean_intvty":143.21862},{"ep":8,"hw":"GB300 NVL72","tp":20,"conc":8,"date":"2026-10-26","num_gpus":20,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"1325686.3396226414","dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":2810.45504,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":1911,"duration_seconds":3599.5156,"input_tput_per_gpu":2783.36689,"output_tput_per_gpu":27.08814,"tput_requests_per_sec":0.53085,"avg_isl":104853.71586,"p50_itl":0.0005600000000000001,"p90_itl":0.0006,"mean_itl":0.00057,"p50_e2el":0.5057699999999999,"p50_tpot":0.0005600000000000001,"p50_ttft":"0.15644999999999998","p75_ttft":"0.37083999999999995","p90_e2el":2.82715,"p90_tpot":0.0006,"p90_ttft":"1.4031500000000001","mean_e2el":1.15524,"mean_tpot":0.00057,"mean_ttft":0.5794400000000001,"stdev_isl":55180.4356,"p50_intvty":1796.86129,"p75_intvty":1736.22939,"p90_intvty":1667.14379,"mean_intvty":1741.46963},{"ep":4,"hw":"GB300 NVL72","tp":10,"conc":176,"date":"2026-10-26","num_gpus":20,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"24002885.051886793","dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":50886.11631,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":35575,"duration_seconds":3612.78939,"input_tput_per_gpu":50433.92406,"output_tput_per_gpu":452.19225,"tput_requests_per_sec":9.84884,"avg_isl":102435.50018,"p50_itl":0.00133,"p90_itl":0.00161,"mean_itl":"0.0013700000000000001","p50_e2el":"1.0303499999999999","p50_tpot":0.00133,"p50_ttft":0.41469,"p75_ttft":0.66374,"p90_e2el":3.53852,"p90_tpot":0.00161,"p90_ttft":0.8965,"mean_e2el":"1.7971300000000001","mean_tpot":"0.0013700000000000001","mean_ttft":0.5804600000000001,"stdev_isl":65555.43539,"p50_intvty":750.74187,"p75_intvty":687.7094,"p90_intvty":621.37376,"mean_intvty":727.76314},{"ep":2,"hw":"GB300 NVL72","tp":6,"conc":50,"date":"2026-10-24","num_gpus":22,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":7851272.759433963,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":16644.69825,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"disagg","total_requests":8337,"duration_seconds":3606.06281,"input_tput_per_gpu":16533.22325,"output_tput_per_gpu":111.475,"tput_requests_per_sec":2.33175,"avg_isl":157327.15737,"p50_itl":0.00152,"p90_itl":0.0019,"mean_itl":0.00149,"p50_e2el":3.16725,"p50_tpot":0.00152,"p50_ttft":"1.6991800000000001","p75_ttft":4.1265,"p90_e2el":11.76339,"p90_tpot":0.0019,"p90_ttft":8.3947,"mean_e2el":5.247439999999999,"mean_tpot":0.00149,"mean_ttft":3.42516,"stdev_isl":169664.59721,"p50_intvty":656.12009,"p75_intvty":575.58794,"p90_intvty":525.63459,"mean_intvty":673.34076},{"ep":4,"hw":"GB300 NVL72","tp":12,"conc":264,"date":"2026-10-26","num_gpus":24,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":29820016.61320755,"dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":63218.43522,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":54228,"duration_seconds":3614.75918,"input_tput_per_gpu":62636.18092,"output_tput_per_gpu":582.2543,"tput_requests_per_sec":15.0047,"avg_isl":100205.66947,"p50_itl":0.00163,"p90_itl":0.00195,"mean_itl":0.0017,"p50_e2el":"1.0958299999999999","p50_tpot":0.00163,"p50_ttft":0.31685,"p75_ttft":0.59047,"p90_e2el":4.39393,"p90_tpot":0.00195,"p90_ttft":0.80624,"mean_e2el":2.11906,"mean_tpot":0.0017,"mean_ttft":0.5974400000000001,"stdev_isl":65866.44723,"p50_intvty":615.17023,"p75_intvty":572.26633,"p90_intvty":513.18175,"mean_intvty":587.94314},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-15","num_gpus":48,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"3757033.8660422172","tput_per_gpu":7964.9117960095,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":11954.6402019944,"output_tput_per_gpu":5970.047593017,"p50_itl":3.5304959309869,"mean_itl":3.1924739909993,"p50_e2el":18.9427911620005,"p50_tpot":0.0185587035779,"p50_ttft":1.5304136075429,"mean_e2el":19.8220946133207,"mean_tpot":0.0186262322202,"mean_ttft":2.6798232272705,"p50_intvty":53.8830740952,"mean_intvty":53.6877232163},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":8192,"date":"2026-06-15","num_gpus":28,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4982356.050025849,"tput_per_gpu":10562.5948260548,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":12327.6059249834,"output_tput_per_gpu":9238.8365018583,"p50_itl":6.6126093019266,"mean_itl":5.9884657569642,"p50_e2el":45.1264605454635,"p50_tpot":0.0358247219746,"p50_ttft":"12.791490473551699","mean_e2el":48.7026132673125,"mean_tpot":0.0349526708879,"mean_ttft":16.5302536571748,"p50_intvty":27.9136848769,"mean_intvty":28.610116898},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":333,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":689070.626603632,"tput_per_gpu":1460.8297283997,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":6570.8402333126,"output_tput_per_gpu":822.0784152855,"p50_itl":2.3557459909935,"mean_itl":2.046460177921,"p50_e2el":10.4934929174487,"p50_tpot":0.0083209126973,"p50_ttft":2.5023648465285,"mean_e2el":10.9427820971021,"mean_tpot":0.0087299002983,"mean_ttft":2.9038195939834,"p50_intvty":120.1791241392,"mean_intvty":114.5488454426},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":25,"date":"2026-06-15","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"104334.32389674529","tput_per_gpu":221.1887666611,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":665.1951314045,"output_tput_per_gpu":132.3874937124,"p50_itl":1.8264863929944,"mean_itl":1.5932300670363,"p50_e2el":7.9378351685591,"p50_tpot":0.0065020535374,"p50_ttft":1.5983677944751,"mean_e2el":8.110499807374099,"mean_tpot":0.006800250325100001,"mean_ttft":1.8777839334803,"p50_intvty":153.7975647617,"mean_intvty":147.053410123},{"ep":4,"hw":"GB300 NVL72","tp":12,"conc":128,"date":"2026-10-26","num_gpus":20,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"16813544.311320756","dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":35644.71394,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":26399,"duration_seconds":3604.65157,"input_tput_per_gpu":35297.73177,"output_tput_per_gpu":346.98216,"tput_requests_per_sec":7.32464,"avg_isl":96394.57889,"p50_itl":0.0012,"p90_itl":0.00145,"mean_itl":0.00121,"p50_e2el":0.904,"p50_tpot":0.0012,"p50_ttft":0.28432,"p75_ttft":0.56657,"p90_e2el":3.0452,"p90_tpot":0.00145,"p90_ttft":0.7209500000000001,"mean_e2el":"1.5648499999999999","mean_tpot":0.00121,"mean_ttft":0.45607,"stdev_isl":66263.66112,"p50_intvty":834.73898,"p75_intvty":764.70581,"p90_intvty":688.43176,"mean_intvty":823.57847},{"ep":2,"hw":"GB300 NVL72","tp":6,"conc":72,"date":"2026-10-24","num_gpus":10,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":4245009.028301887,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":8999.41914,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"disagg","total_requests":2554,"duration_seconds":3611.64466,"input_tput_per_gpu":8925.88006,"output_tput_per_gpu":73.53907,"tput_requests_per_sec":0.70701,"avg_isl":126222.0325,"p50_itl":0.00163,"p90_itl":0.00237,"mean_itl":"0.0016799999999999999","p50_e2el":"114.66278999999999","p50_tpot":0.00163,"p50_ttft":113.26867,"p75_ttft":157.90903,"p90_e2el":"206.54339000000002","p90_tpot":0.00237,"p90_ttft":204.34692,"mean_e2el":105.86325,"mean_tpot":"0.0016799999999999999","mean_ttft":103.91135,"stdev_isl":134673.55459,"p50_intvty":615.15006,"p75_intvty":493.29429,"p90_intvty":421.17955,"mean_intvty":596.26193},{"ep":8,"hw":"GB300 NVL72","tp":20,"conc":16,"date":"2026-10-26","num_gpus":20,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":2544910.306603774,"dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":5395.20985,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":3530,"duration_seconds":3599.69682,"input_tput_per_gpu":5349.61769,"output_tput_per_gpu":45.59216,"tput_requests_per_sec":0.98055,"avg_isl":109104.82606,"p50_itl":0.00065,"p90_itl":0.00076,"mean_itl":0.00067,"p50_e2el":0.47158,"p50_tpot":0.00065,"p50_ttft":0.15632,"p75_ttft":0.25461,"p90_e2el":2.25578,"p90_tpot":0.00076,"p90_ttft":0.64143,"mean_e2el":0.93128,"mean_tpot":0.00067,"mean_ttft":"0.33197000000000004","stdev_isl":61078.75438,"p50_intvty":1539.55894,"p75_intvty":1372.17605,"p90_intvty":1317.29465,"mean_intvty":1497.62537},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":667445.6999467154,"tput_per_gpu":914.400608927,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":456.4760368772,"output_tput_per_gpu":457.9245720498,"p50_itl":0.0152980890125,"p90_itl":"0.015873886644800002","mean_itl":"0.016889833386500002","p50_e2el":15.756152363494,"p50_tpot":0.0170259808717,"p50_ttft":"0.10365770012139999","p90_e2el":17.3711880164221,"p90_tpot":0.0173804414109,"p90_ttft":0.261076760944,"mean_e2el":15.7515925351792,"mean_tpot":0.0168827694454,"mean_ttft":0.1675017594534,"p50_intvty":58.7337673838,"p90_intvty":57.5359380326,"mean_intvty":59.2319881662},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":615,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1166890.7648407076","tput_per_gpu":2473.8084214623,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":11133.3020276572,"output_tput_per_gpu":1391.3717206879,"p50_itl":2.4630343429744,"mean_itl":2.1417539539614,"p50_e2el":11.2581190629862,"p50_tpot":0.008690328326999999,"p50_ttft":"3.0138519444735996","mean_e2el":"11.850302527694101","mean_tpot":0.0091157498428,"mean_ttft":3.4574131574339,"p50_intvty":115.0704510085,"mean_intvty":109.700245975},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":1229,"date":"2026-06-15","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1991748.7501817925","tput_per_gpu":4222.5073503854,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":10558.9113354266,"output_tput_per_gpu":2638.4063541251,"p50_itl":2.9104698229348,"mean_itl":2.5107427418633,"p50_e2el":"11.935332480003101","p50_tpot":0.0102077904718,"p50_ttft":"2.1814305121079003","mean_e2el":12.5770693404617,"mean_tpot":0.0107203529707,"mean_ttft":2.7077462999506,"p50_intvty":97.964393251,"mean_intvty":93.2805107008},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":896,"date":"2026-03-23","num_gpus":64,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":721525.2967091971,"tput_per_gpu":988.4896564916,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":3953.3121321026,"output_tput_per_gpu":564.9435885472,"p50_itl":5.132950935047099,"mean_itl":4.5345815648242,"p50_e2el":24.0748594070319,"p50_tpot":0.0217920997707,"p50_ttft":4.0113172403071,"mean_e2el":"24.846131647857298","mean_tpot":0.0211989099647,"mean_ttft":5.3357909864089,"p50_intvty":45.8881893218,"mean_intvty":47.1722367643},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":48,"date":"2026-03-23","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":560918.9472044526,"tput_per_gpu":768.4589576701,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2730.966259779,"output_tput_per_gpu":114.2898569672,"p50_itl":2.8706050440669,"mean_itl":2.5067171161495,"p50_e2el":15.5981510159327,"p50_tpot":0.0128635035437,"p50_ttft":3.7256986449938,"mean_e2el":15.6594304780204,"mean_tpot":0.0123561457489,"mean_ttft":4.2225340591792,"p50_intvty":77.7393185773,"mean_intvty":80.9313859129},{"hw":"B300","tp":1,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-04","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":569127.7519297894,"tput_per_gpu":1081.3427286666,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":138.9551934959},{"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-04","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":94359.39031457895,"tput_per_gpu":179.2828415977,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":186.8057477814},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":48,"date":"2026-10-27","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"5562174.4433962265","tput_per_gpu":11791.80982,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":6690,"num_prefill_gpu":8,"duration_seconds":3629.90638,"input_tput_per_gpu":11712.57298,"total_input_tokens":1021715840,"output_tput_per_gpu":79.23685,"total_output_tokens":6918570,"avg_output_tokens_expected":1234.97275,"theoretical_cache_hit_rate":0.97454,"stdev_output_tokens_expected":1994.35433,"avg_isl":152522.12885,"avg_osl":1031.829,"p50_itl":0.00749,"p75_isl":199418.25,"p75_osl":1052.5,"p75_qps":3.0,"p90_isl":326770.2,"p90_itl":0.00932,"p90_osl":2471.3,"p90_qps":4.0,"p95_isl":563074.25,"p95_osl":3985.3,"p95_qps":4.0,"std_qps":1.44751,"mean_itl":"0.0074800000000000005","mean_qps":1.84321,"p50_e2el":4.644399999999999,"p50_tpot":0.00749,"p50_ttft":0.89358,"p90_e2el":20.38937,"p90_tpot":0.00932,"p90_ttft":2.4989,"mean_e2el":9.235,"mean_tpot":0.00752,"mean_ttft":"1.3449200000000001","stdev_isl":162059.07138,"stdev_osl":2100.80955,"median_qps":2.0,"p50_intvty":133.5113484646,"p90_intvty":107.2961373391,"mean_intvty":133.6898395722},{"ep":2,"hw":"GB300 NVL72","tp":6,"conc":54,"date":"2026-10-24","num_gpus":14,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"13193506.745283019","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":27970.2343,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"disagg","total_requests":9226,"duration_seconds":3617.73794,"input_tput_per_gpu":27778.90241,"output_tput_per_gpu":191.33189,"tput_requests_per_sec":2.5499,"avg_isl":152498.92142,"p50_itl":0.00197,"p90_itl":0.0025,"mean_itl":0.00191,"p50_e2el":3.50288,"p50_tpot":0.00197,"p50_ttft":"1.5621800000000001","p75_ttft":4.68478,"p90_e2el":"13.963809999999999","p90_tpot":0.0025,"p90_ttft":"10.159360000000001","mean_e2el":6.01628,"mean_tpot":0.00191,"mean_ttft":"3.7164800000000002","stdev_isl":167024.11836,"p50_intvty":506.97789,"p75_intvty":433.22775,"p90_intvty":399.68351,"mean_intvty":524.17187},{"ep":1,"hw":"GB300 NVL72","tp":2,"conc":35,"date":"2026-10-24","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":47499642.58490566,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":100699.24228,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"agg","total_requests":4496,"duration_seconds":3610.66538,"input_tput_per_gpu":100034.13149,"output_tput_per_gpu":665.11079,"tput_requests_per_sec":1.24557,"avg_isl":160671.60832,"p50_itl":0.00788,"p90_itl":0.01188,"mean_itl":0.00811,"p50_e2el":4.46453,"p50_tpot":0.00788,"p50_ttft":0.7293200000000001,"p75_ttft":0.93855,"p90_e2el":"25.438119999999998","p90_tpot":0.01188,"p90_ttft":1.35323,"mean_e2el":10.9215,"mean_tpot":0.00811,"mean_ttft":0.91404,"stdev_isl":138793.63695,"p50_intvty":126.86941,"p75_intvty":100.92758,"p90_intvty":84.18194,"mean_intvty":123.2286},{"ep":4,"hw":"GB300 NVL72","tp":12,"conc":28,"date":"2026-10-26","num_gpus":12,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":6692117.471698114,"dataset_name":"semianalysisai/cc-traces-weka-062126-256k","tput_per_gpu":14187.28904,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"disagg","total_requests":5601,"duration_seconds":3599.08579,"input_tput_per_gpu":14061.3521,"output_tput_per_gpu":125.93694,"tput_requests_per_sec":1.55642,"avg_isl":108426.37922,"p50_itl":0.00091,"p90_itl":0.00114,"mean_itl":0.00094,"p50_e2el":0.62567,"p50_tpot":0.00091,"p50_ttft":0.14353,"p75_ttft":0.28,"p90_e2el":2.98626,"p90_tpot":0.00114,"p90_ttft":0.94956,"mean_e2el":1.2634,"mean_tpot":0.00094,"mean_ttft":0.37861,"stdev_isl":65299.2544,"p50_intvty":1101.23224,"p75_intvty":921.13007,"p90_intvty":873.47466,"mean_intvty":1067.14579},{"ep":2,"hw":"GB300 NVL72","tp":10,"conc":12,"date":"2026-10-24","num_gpus":18,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"1771993.4056603776","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":3756.62602,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"disagg","total_requests":1708,"duration_seconds":3599.7145,"input_tput_per_gpu":3728.03144,"output_tput_per_gpu":28.59458,"tput_requests_per_sec":0.4743,"avg_isl":141426.97834,"p50_itl":0.00106,"p90_itl":0.00128,"mean_itl":0.00102,"p50_e2el":"1.5504200000000001","p50_tpot":0.00106,"p50_ttft":0.74272,"p75_ttft":1.77367,"p90_e2el":8.07783,"p90_tpot":0.00128,"p90_ttft":4.77574,"mean_e2el":3.31929,"mean_tpot":0.00102,"mean_ttft":1.98907,"stdev_isl":115860.92252,"p50_intvty":944.85644,"p75_intvty":843.80636,"p90_intvty":781.42239,"mean_intvty":981.30028},{"ep":1,"hw":"GB300 NVL72","tp":2,"conc":30,"date":"2026-10-24","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":41762465.02358491,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":88536.42585,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"agg","total_requests":4043,"duration_seconds":3614.57227,"input_tput_per_gpu":87971.43964,"output_tput_per_gpu":564.98621,"tput_requests_per_sec":1.11936,"avg_isl":157298.60326,"p50_itl":0.00658,"p90_itl":0.01031,"mean_itl":0.006849999999999999,"p50_e2el":3.75583,"p50_tpot":0.00658,"p50_ttft":0.6335299999999999,"p75_ttft":0.8141,"p90_e2el":"20.919919999999998","p90_tpot":0.01031,"p90_ttft":1.14114,"mean_e2el":8.78936,"mean_tpot":0.006849999999999999,"mean_ttft":0.7937000000000001,"stdev_isl":131809.02561,"p50_intvty":151.99645,"p75_intvty":120.79769,"p90_intvty":96.98055,"mean_intvty":145.97488},{"ep":1,"hw":"GB300 NVL72","tp":2,"conc":25,"date":"2026-10-24","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":37875203.58962264,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":80295.43161,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"agg","total_requests":3745,"duration_seconds":3619.03245,"input_tput_per_gpu":79785.14288,"output_tput_per_gpu":510.28874,"tput_requests_per_sec":1.03568,"avg_isl":154202.94873,"p50_itl":0.00588,"p90_itl":0.008919999999999999,"mean_itl":"0.0060999999999999995","p50_e2el":3.11341,"p50_tpot":0.00588,"p50_ttft":0.57298,"p75_ttft":0.7327400000000001,"p90_e2el":19.23836,"p90_tpot":0.008919999999999999,"p90_ttft":1.1554,"mean_e2el":7.82641,"mean_tpot":"0.0060999999999999995","mean_ttft":0.76852,"stdev_isl":134420.50683,"p50_intvty":170.20844,"p75_intvty":134.79398,"p90_intvty":112.08672,"mean_intvty":163.98229},{"ep":1,"hw":"GB300 NVL72","tp":2,"conc":20,"date":"2026-10-24","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"30913233.424528304","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":65536.05486,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"agg","total_requests":2983,"duration_seconds":3606.80458,"input_tput_per_gpu":65101.15925,"output_tput_per_gpu":434.89562,"tput_requests_per_sec":0.82741,"avg_isl":157430.21086,"p50_itl":0.00511,"p90_itl":0.007730000000000001,"mean_itl":0.0053,"p50_e2el":2.84814,"p50_tpot":0.00511,"p50_ttft":0.5159199999999999,"p75_ttft":0.65763,"p90_e2el":17.91977,"p90_tpot":0.007730000000000001,"p90_ttft":0.93652,"mean_e2el":7.04835,"mean_tpot":0.0053,"mean_ttft":0.6593,"stdev_isl":144076.61911,"p50_intvty":195.56627,"p75_intvty":157.65199,"p90_intvty":129.30494,"mean_intvty":188.76472},{"ep":1,"hw":"GB300 NVL72","tp":2,"conc":45,"date":"2026-10-24","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":53288713.08962265,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":112972.07175,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792825200,"serving_config":"agg","total_requests":5234,"duration_seconds":3626.66068,"input_tput_per_gpu":112228.33068,"output_tput_per_gpu":743.74107,"tput_requests_per_sec":1.44319,"avg_isl":155526.96752,"p50_itl":0.01022,"p90_itl":0.01506,"mean_itl":0.01041,"p50_e2el":"6.0738900000000005","p50_tpot":0.01022,"p50_ttft":0.91154,"p75_ttft":1.33614,"p90_e2el":31.79993,"p90_tpot":0.01506,"p90_ttft":2.47843,"mean_e2el":13.80091,"mean_tpot":0.01041,"mean_ttft":1.44845,"stdev_isl":160340.2414,"p50_intvty":97.84262,"p75_intvty":79.62004,"p90_intvty":66.41741,"mean_intvty":96.10262},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":4096,"date":"2026-06-03","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5487707.707961368,"tput_per_gpu":11633.9403408781,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":13788.8367866001,"output_tput_per_gpu":5169.2510037122,"p50_itl":"1.7095668970142002","mean_itl":3.5249994032292,"p50_e2el":84.8215044959215,"p50_tpot":0.0226665245235,"p50_ttft":62.4801501650363,"mean_e2el":84.404328815736,"mean_tpot":"0.024277985922799998","mean_ttft":62.0613167106897,"p50_intvty":44.1179237233,"mean_intvty":41.1895782122},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":240,"date":"2026-10-27","num_gpus":56,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":9609098.740566038,"tput_per_gpu":20371.28933,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":32,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":29635,"num_prefill_gpu":24,"duration_seconds":3629.67706,"input_tput_per_gpu":20225.46522,"total_input_tokens":4119266314,"output_tput_per_gpu":145.82411,"total_output_tokens":29895887,"avg_output_tokens_expected":1325.9577,"theoretical_cache_hit_rate":0.97055,"stdev_output_tokens_expected":2348.40252,"avg_isl":138723.36086,"avg_osl":1000.18519,"p50_itl":"0.013359999999999999","p75_isl":170655.5,"p75_osl":964.0,"p75_qps":10.0,"p90_isl":287461.2,"p90_itl":0.01467,"p90_osl":2296.6,"p90_qps":12.0,"p95_isl":424185.6,"p95_osl":3846.3,"p95_qps":14.0,"std_qps":3.19744,"mean_itl":"0.012960000000000001","mean_qps":8.16561,"p50_e2el":7.20104,"p50_tpot":"0.013359999999999999","p50_ttft":1.24201,"p90_e2el":33.62478,"p90_tpot":0.01466,"p90_ttft":"3.0001700000000002","mean_e2el":"15.246129999999999","mean_tpot":0.01302,"mean_ttft":1.89049,"stdev_isl":134476.74311,"stdev_osl":2058.31234,"median_qps":8.0,"p50_intvty":74.8502994012,"p90_intvty":68.166325835,"mean_intvty":77.1604938272},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":64,"date":"2026-10-27","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":7227205.70754717,"tput_per_gpu":15321.6761,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":9773,"num_prefill_gpu":8,"duration_seconds":3626.68379,"input_tput_per_gpu":15210.80035,"total_input_tokens":1326089086,"output_tput_per_gpu":110.87575,"total_output_tokens":9688566,"avg_output_tokens_expected":1231.56077,"theoretical_cache_hit_rate":0.97076,"stdev_output_tokens_expected":2169.81285,"avg_isl":135470.61424,"avg_osl":987.48296,"p50_itl":0.009880000000000002,"p75_isl":165519.0,"p75_osl":1050.0,"p75_qps":4.0,"p90_isl":295983.0,"p90_itl":"0.011619999999999998","p90_osl":2232.6,"p90_qps":5.0,"p95_isl":511108.4,"p95_osl":3670.4,"p95_qps":6.0,"std_qps":1.80676,"mean_itl":0.00966,"mean_qps":2.69498,"p50_e2el":6.40164,"p50_tpot":0.009890000000000001,"p50_ttft":1.1807,"p90_e2el":24.82246,"p90_tpot":"0.011609999999999999","p90_ttft":3.64182,"mean_e2el":11.67403,"mean_tpot":0.009720000000000001,"mean_ttft":1.94546,"stdev_isl":154391.20839,"stdev_osl":1910.60218,"median_qps":2.0,"p50_intvty":101.2145748988,"p90_intvty":86.0585197935,"mean_intvty":103.5196687371},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":240,"date":"2026-10-27","num_gpus":40,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"11503497.891509434","tput_per_gpu":24387.41553,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":24,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":25871,"num_prefill_gpu":16,"duration_seconds":3628.9559,"input_tput_per_gpu":24209.80974,"total_input_tokens":3524111837,"output_tput_per_gpu":177.6058,"total_output_tokens":26051241,"avg_output_tokens_expected":1333.32808,"theoretical_cache_hit_rate":0.96946,"stdev_output_tokens_expected":2371.97384,"avg_isl":135837.55081,"avg_osl":996.51904,"p50_itl":0.01563,"p75_isl":166256.5,"p75_osl":962.0,"p75_qps":9.0,"p90_isl":283308.0,"p90_itl":0.01715,"p90_osl":2301.0,"p90_qps":11.0,"p95_isl":418481.0,"p95_osl":3833.0,"p95_qps":12.0,"std_qps":3.10981,"mean_itl":"0.015009999999999999","mean_qps":7.13065,"p50_e2el":"11.659889999999999","p50_tpot":0.01563,"p50_ttft":"3.5989400000000002","p90_e2el":44.11667,"p90_tpot":0.01714,"p90_ttft":"10.697899999999999","mean_e2el":"20.722099999999998","mean_tpot":0.01509,"mean_ttft":"5.3600699999999994","stdev_isl":133536.57167,"stdev_osl":2038.96832,"median_qps":7.0,"p50_intvty":63.9795265515,"p90_intvty":58.3090379009,"mean_intvty":66.6222518321},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":32,"date":"2026-10-27","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":4535844.339622642,"tput_per_gpu":9615.99,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":4649,"num_prefill_gpu":8,"duration_seconds":3614.32349,"input_tput_per_gpu":9560.28457,"total_input_tokens":829722466,"output_tput_per_gpu":55.70543,"total_output_tokens":4832101,"avg_output_tokens_expected":1399.5116,"theoretical_cache_hit_rate":0.97993,"stdev_output_tokens_expected":2133.43389,"avg_isl":178381.38675,"avg_osl":1039.3846,"p50_itl":0.00392,"p75_isl":237194.0,"p75_osl":1119.0,"p75_qps":2.0,"p90_isl":353599.6,"p90_itl":0.00598,"p90_osl":2656.6,"p90_qps":3.0,"p95_isl":517298.2,"p95_osl":4162.6,"p95_qps":4.0,"std_qps":1.23141,"mean_itl":0.00421,"mean_qps":1.28611,"p50_e2el":"2.9908200000000003","p50_tpot":0.00392,"p50_ttft":0.84224,"p90_e2el":"12.952879999999999","p90_tpot":0.00597,"p90_ttft":2.1447,"mean_e2el":5.76584,"mean_tpot":0.00423,"mean_ttft":1.32892,"stdev_isl":159212.98603,"stdev_osl":1720.33078,"median_qps":1.0,"p50_intvty":255.1020408163,"p90_intvty":167.2240802676,"mean_intvty":237.5296912114},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":336,"date":"2026-10-27","num_gpus":40,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"12530448.452830188","tput_per_gpu":26564.55072,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":31619,"num_prefill_gpu":24,"duration_seconds":3629.59894,"input_tput_per_gpu":26358.7009,"total_input_tokens":3854726043,"output_tput_per_gpu":205.84982,"total_output_tokens":30339423,"avg_output_tokens_expected":1275.69243,"theoretical_cache_hit_rate":0.96615,"stdev_output_tokens_expected":2430.64185,"avg_isl":121030.40944,"avg_osl":945.19406,"p50_itl":0.027,"p75_isl":147254.5,"p75_osl":971.0,"p75_qps":11.0,"p90_isl":253870.0,"p90_itl":0.02925,"p90_osl":2161.2,"p90_qps":13.0,"p95_isl":395004.5,"p95_osl":3601.1,"p95_qps":14.6,"std_qps":3.307,"mean_itl":"0.026010000000000002","mean_qps":8.71232,"p50_e2el":14.50431,"p50_tpot":0.02699,"p50_ttft":1.80931,"p90_e2el":"63.215489999999996","p90_tpot":0.02924,"p90_ttft":7.08423,"mean_e2el":"28.941380000000002","mean_tpot":0.02614,"mean_ttft":3.53166,"stdev_isl":127858.62446,"stdev_osl":1812.88868,"median_qps":8.0,"p50_intvty":37.037037037,"p90_intvty":34.188034188,"mean_intvty":38.4467512495},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":16,"date":"2026-10-27","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1675596.1179245284","tput_per_gpu":3552.26377,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":24,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":2635,"num_prefill_gpu":8,"duration_seconds":3616.41633,"input_tput_per_gpu":3529.82194,"total_input_tokens":409045497,"output_tput_per_gpu":22.44183,"total_output_tokens":2597090,"avg_output_tokens_expected":1612.21111,"theoretical_cache_hit_rate":0.97614,"stdev_output_tokens_expected":2505.30901,"avg_isl":155024.58558,"avg_osl":985.61214,"p50_itl":0.0029,"p75_isl":183500.5,"p75_osl":940.5,"p75_qps":1.0,"p90_isl":279914.8,"p90_itl":"0.0033599999999999997","p90_osl":2421.4,"p90_qps":2.0,"p95_isl":525608.9,"p95_osl":4215.3,"p95_qps":3.0,"std_qps":0.91378,"mean_itl":"0.0029500000000000004","mean_qps":0.72843,"p50_e2el":1.906,"p50_tpot":0.0029,"p50_ttft":0.57228,"p90_e2el":8.342139999999999,"p90_tpot":0.00335,"p90_ttft":1.50424,"mean_e2el":3.76236,"mean_tpot":"0.0029500000000000004","mean_ttft":0.80414,"stdev_isl":152201.14193,"stdev_osl":1934.09233,"median_qps":0.0,"p50_intvty":344.8275862069,"p90_intvty":297.619047619,"mean_intvty":338.9830508475},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":24,"date":"2026-10-27","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"3502964.4811320757","tput_per_gpu":7426.2847,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":3842,"num_prefill_gpu":8,"duration_seconds":3604.3083,"input_tput_per_gpu":7381.57542,"total_input_tokens":638915465,"output_tput_per_gpu":44.70927,"total_output_tokens":3867505,"avg_output_tokens_expected":1504.19622,"theoretical_cache_hit_rate":0.97899,"stdev_output_tokens_expected":2256.20441,"avg_isl":166197.64836,"avg_osl":1006.63821,"p50_itl":0.00342,"p75_isl":221622.0,"p75_osl":1038.75,"p75_qps":2.0,"p90_isl":331233.5,"p90_itl":0.00453,"p90_osl":2607.1,"p90_qps":3.0,"p95_isl":410275.2,"p95_osl":4211.5,"p95_qps":3.0,"std_qps":1.11933,"mean_itl":0.00357,"mean_qps":1.06576,"p50_e2el":"2.2958000000000003","p50_tpot":0.00342,"p50_ttft":0.70904,"p90_e2el":"11.146780000000001","p90_tpot":0.00452,"p90_ttft":1.70425,"mean_e2el":4.732,"mean_tpot":"0.0035800000000000003","mean_ttft":"1.0666099999999998","stdev_isl":143804.44252,"stdev_osl":1761.82447,"median_qps":1.0,"p50_intvty":292.3976608187,"p90_intvty":220.7505518764,"mean_intvty":280.1120448179},{"ep":1,"hw":"Vera Rubin NVL72","tp":16,"conc":48,"date":"2026-10-23","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":3566468.424242424,"tput_per_gpu":11769.3458,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":8394,"num_prefill_gpu":16,"duration_seconds":3621.81905,"input_tput_per_gpu":11692.6453,"total_input_tokens":1355540754,"output_tput_per_gpu":76.7005,"total_output_tokens":8889452,"avg_output_tokens_expected":1295.99632,"theoretical_cache_hit_rate":0.97652,"stdev_output_tokens_expected":2190.27508,"avg_isl":161443.48999,"avg_osl":1059.02442,"p50_itl":0.00239,"p75_isl":212939.25,"p75_osl":1080.75,"p75_qps":3.0,"p90_isl":364571.8,"p90_itl":0.00292,"p90_osl":2547.7,"p90_qps":4.0,"p95_isl":605393.9,"p95_osl":4094.7,"p95_qps":5.0,"std_qps":1.63444,"mean_itl":0.00237,"mean_qps":2.31787,"p50_e2el":"1.4151500000000001","p50_tpot":"0.0023799999999999997","p50_ttft":0.24768,"p90_e2el":"6.5305100000000005","p90_tpot":"0.0029100000000000003","p90_ttft":0.5482899999999999,"mean_e2el":"2.8961900000000003","mean_tpot":0.00237,"mean_ttft":0.33213,"stdev_isl":175487.26723,"stdev_osl":2089.04172,"median_qps":2.0,"p50_intvty":418.410041841,"p90_intvty":342.4657534247,"mean_intvty":421.94092827},{"ep":1,"hw":"Vera Rubin NVL72","tp":16,"conc":1,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"411908.39090909093","tput_per_gpu":1359.29769,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":438,"num_prefill_gpu":8,"duration_seconds":3594.80169,"input_tput_per_gpu":1350.97396,"total_input_tokens":117589116,"output_tput_per_gpu":8.32373,"total_output_tokens":718134,"avg_output_tokens_expected":2119.93607,"theoretical_cache_hit_rate":0.98266,"stdev_output_tokens_expected":2576.30726,"avg_isl":266108.68265,"avg_osl":1639.57078,"p50_itl":0.00108,"p75_isl":433350.0,"p75_osl":2249.75,"p75_qps":0.0,"p90_isl":583584.6,"p90_itl":0.00116,"p90_osl":4782.0,"p90_qps":1.0,"p95_isl":643616.1,"p95_osl":6031.5,"p95_qps":1.0,"std_qps":0.34749,"mean_itl":0.00107,"mean_qps":0.12163,"p50_e2el":1.01164,"p50_tpot":0.00107,"p50_ttft":0.23469,"p90_e2el":"5.5425699999999996","p90_tpot":0.00115,"p90_ttft":0.37338,"mean_e2el":2.08504,"mean_tpot":0.00106,"mean_ttft":"0.27132999999999996","stdev_isl":211768.51254,"stdev_osl":2338.22019,"median_qps":0.0,"p50_intvty":925.9259259259,"p90_intvty":862.0689655172,"mean_intvty":934.5794392523},{"ep":1,"hw":"GB300 NVL72","tp":16,"conc":1,"date":"2026-10-27","num_gpus":16,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":735009.2500000001,"tput_per_gpu":1558.21961,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793084400,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40239967341/attempts/1","total_requests":296,"duration_seconds":3594.27369,"input_tput_per_gpu":1548.09681,"total_input_tokens":89439952,"output_tput_per_gpu":10.1228,"total_output_tokens":582801,"avg_output_tokens_expected":2125.15878,"theoretical_cache_hit_rate":0.98241,"stdev_output_tokens_expected":2805.83643,"avg_isl":300772.08784,"avg_osl":1966.70946,"p50_itl":0.00219,"p75_isl":436282.75,"p75_osl":2693.0,"p75_qps":0.0,"p90_isl":544789.5,"p90_itl":"0.0023599999999999997","p90_osl":5082.5,"p90_qps":0.0,"p95_isl":572144.5,"p95_osl":6696.75,"p95_qps":1.0,"std_qps":0.27965,"mean_itl":0.00219,"mean_qps":0.08217,"p50_e2el":"1.5155399999999999","p50_tpot":"0.0020499999999999997","p50_ttft":0.72424,"p90_e2el":8.3437,"p90_tpot":"0.0022299999999999998","p90_ttft":1.29098,"mean_e2el":3.2183,"mean_tpot":0.00147,"mean_ttft":0.83604,"stdev_isl":175607.5237,"stdev_osl":2552.71732,"median_qps":0.0,"p50_intvty":456.6210045662,"p90_intvty":423.7288135593,"mean_intvty":456.6210045662},{"ep":1,"hw":"Vera Rubin NVL72","tp":16,"conc":160,"date":"2026-10-23","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":9607527.56969697,"tput_per_gpu":31704.84098,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":24525,"num_prefill_gpu":16,"duration_seconds":3616.22158,"input_tput_per_gpu":31479.10282,"total_input_tokens":3644240855,"output_tput_per_gpu":225.73815,"total_output_tokens":26122221,"avg_output_tokens_expected":1338.64863,"theoretical_cache_hit_rate":0.9729,"stdev_output_tokens_expected":2511.37365,"avg_isl":148531.4232,"avg_osl":1065.12595,"p50_itl":0.00495,"p75_isl":190172.0,"p75_osl":1018.0,"p75_qps":9.0,"p90_isl":315791.4,"p90_itl":0.00584,"p90_osl":2453.6,"p90_qps":11.0,"p95_isl":462757.2,"p95_osl":4105.8,"p95_qps":12.0,"std_qps":2.98328,"mean_itl":0.00491,"mean_qps":6.78208,"p50_e2el":3.02788,"p50_tpot":0.004940000000000001,"p50_ttft":0.63019,"p90_e2el":13.53849,"p90_tpot":0.00583,"p90_ttft":1.52996,"mean_e2el":6.220680000000001,"mean_tpot":0.00492,"mean_ttft":0.88739,"stdev_isl":146716.88448,"stdev_osl":2150.02482,"median_qps":7.0,"p50_intvty":202.0202020202,"p90_intvty":171.2328767123,"mean_intvty":203.66598778},{"ep":1,"hw":"Vera Rubin NVL72","tp":16,"conc":256,"date":"2026-10-23","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"13562143.136363637","tput_per_gpu":44755.07235,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":36207,"num_prefill_gpu":16,"duration_seconds":3629.38562,"input_tput_per_gpu":44442.34824,"total_input_tokens":5165591440,"output_tput_per_gpu":312.72412,"total_output_tokens":36442389,"avg_output_tokens_expected":1298.94239,"theoretical_cache_hit_rate":0.97171,"stdev_output_tokens_expected":2328.27729,"avg_isl":142556.67208,"avg_osl":1003.11777,"p50_itl":0.00824,"p75_isl":181396.5,"p75_osl":997.0,"p75_qps":12.0,"p90_isl":298210.4,"p90_itl":0.00977,"p90_osl":2335.4,"p90_qps":15.0,"p95_isl":430262.8,"p95_osl":3836.0,"p95_qps":16.0,"std_qps":3.84008,"mean_itl":0.00807,"mean_qps":9.97685,"p50_e2el":5.19384,"p50_tpot":0.00822,"p50_ttft":1.19108,"p90_e2el":22.75084,"p90_tpot":0.00973,"p90_ttft":"3.4901999999999997","mean_e2el":10.23766,"mean_tpot":0.00807,"mean_ttft":1.96186,"stdev_isl":138174.91355,"stdev_osl":2052.57254,"median_qps":10.0,"p50_intvty":121.359223301,"p90_intvty":102.3541453429,"mean_intvty":123.9157372986},{"ep":1,"hw":"Vera Rubin NVL72","tp":16,"conc":64,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":6416380.621212121,"tput_per_gpu":21174.05605,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":12108,"num_prefill_gpu":8,"duration_seconds":3603.51144,"input_tput_per_gpu":21028.55648,"total_input_tokens":1819701294,"output_tput_per_gpu":145.49957,"total_output_tokens":12583429,"avg_output_tokens_expected":1313.56341,"theoretical_cache_hit_rate":0.97435,"stdev_output_tokens_expected":2349.31716,"avg_isl":150201.47423,"avg_osl":1039.26536,"p50_itl":0.00315,"p75_isl":194399.5,"p75_osl":1104.25,"p75_qps":5.0,"p90_isl":353924.5,"p90_itl":0.004,"p90_osl":2397.3,"p90_qps":6.0,"p95_isl":560743.6,"p95_osl":3919.55,"p95_qps":7.0,"std_qps":2.08689,"mean_itl":"0.0030800000000000003","mean_qps":3.36119,"p50_e2el":2.0187,"p50_tpot":0.00315,"p50_ttft":0.33096,"p90_e2el":8.497530000000001,"p90_tpot":0.004,"p90_ttft":1.15235,"mean_e2el":"3.8695399999999998","mean_tpot":0.00309,"mean_ttft":0.6088300000000001,"stdev_isl":170499.66829,"stdev_osl":1964.66434,"median_qps":3.0,"p50_intvty":317.4603174603,"p90_intvty":250.0,"mean_intvty":324.6753246753},{"ep":1,"hw":"Vera Rubin NVL72","tp":16,"conc":8,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":16,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":908503.4363636364,"tput_per_gpu":2998.06134,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":1481,"num_prefill_gpu":8,"duration_seconds":3596.45868,"input_tput_per_gpu":2977.84449,"total_input_tokens":258237796,"output_tput_per_gpu":20.21684,"total_output_tokens":1745020,"avg_output_tokens_expected":1930.09091,"theoretical_cache_hit_rate":0.98025,"stdev_output_tokens_expected":2458.81727,"avg_isl":173553.45847,"avg_osl":1178.26941,"p50_itl":0.00116,"p75_isl":210351.0,"p75_osl":1254.0,"p75_qps":1.0,"p90_isl":442025.0,"p90_itl":"0.0016899999999999999","p90_osl":3010.0,"p90_qps":1.0,"p95_isl":543494.0,"p95_osl":4596.0,"p95_qps":2.0,"std_qps":0.64991,"mean_itl":0.00126,"mean_qps":0.41157,"p50_e2el":0.84005,"p50_tpot":0.00116,"p50_ttft":"0.17028000000000001","p90_e2el":4.22833,"p90_tpot":"0.0016799999999999999","p90_ttft":0.33704,"mean_e2el":1.76142,"mean_tpot":0.00125,"mean_ttft":0.22707,"stdev_isl":154851.11451,"stdev_osl":1909.77851,"median_qps":0.0,"p50_intvty":862.0689655172,"p90_intvty":591.7159763314,"mean_intvty":793.6507936508},{"ep":8,"hw":"B300","pp":1,"tp":8,"conc":48,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":12367762.7,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":23498.74913,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":4743,"duration_seconds":3627.25099,"input_tput_per_gpu":23337.5962,"output_tput_per_gpu":161.15293,"tput_requests_per_sec":1.30778,"avg_isl":142781.0565,"p50_itl":0.0198,"p90_itl":0.02711,"mean_itl":"0.020989999999999998","p50_e2el":9.25009,"p50_tpot":0.0198,"p50_ttft":0.54896,"p75_ttft":0.8256100000000001,"p90_e2el":48.38946,"p90_tpot":0.02711,"p90_ttft":1.40795,"mean_e2el":21.12101,"mean_tpot":"0.020989999999999998","mean_ttft":0.85505,"stdev_isl":147170.94954,"p50_intvty":50.49546,"p75_intvty":43.80667,"p90_intvty":36.88022,"mean_intvty":47.65119},{"ep":1,"hw":"B300","pp":1,"tp":8,"conc":8,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"3679862.6210526316","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":6991.73898,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":1321,"duration_seconds":3610.38727,"input_tput_per_gpu":6940.63608,"output_tput_per_gpu":51.1029,"tput_requests_per_sec":0.36565,"avg_isl":151754.02952,"p50_itl":0.00436,"p90_itl":0.00719,"mean_itl":0.00511,"p50_e2el":"2.6726300000000003","p50_tpot":0.00436,"p50_ttft":"0.38001999999999997","p75_ttft":0.55149,"p90_e2el":"13.615219999999999","p90_tpot":0.00719,"p90_ttft":0.76958,"mean_e2el":5.58256,"mean_tpot":0.00511,"mean_ttft":0.54648,"stdev_isl":128950.87508,"p50_intvty":229.33571,"p75_intvty":175.53899,"p90_intvty":139.0957,"mean_intvty":195.76394},{"ep":8,"hw":"B300","pp":1,"tp":8,"conc":32,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":9792260.757894738,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":18605.29544,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":3511,"duration_seconds":3617.62355,"input_tput_per_gpu":18485.89243,"output_tput_per_gpu":119.40301,"tput_requests_per_sec":0.97122,"avg_isl":152378.23925,"p50_itl":0.01478,"p90_itl":0.02255,"mean_itl":0.0166,"p50_e2el":6.8485,"p50_tpot":0.01478,"p50_ttft":0.43697,"p75_ttft":0.621,"p90_e2el":"39.710989999999995","p90_tpot":0.02255,"p90_ttft":1.05478,"mean_e2el":16.09806,"mean_tpot":0.0166,"mean_ttft":0.67195,"stdev_isl":137693.51551,"p50_intvty":67.65912,"p75_intvty":55.98622,"p90_intvty":44.34366,"mean_intvty":60.25679},{"ep":1,"hw":"B300","pp":1,"tp":8,"conc":16,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":6185673.878947369,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":11752.78037,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":2363,"duration_seconds":3626.72278,"input_tput_per_gpu":11673.13036,"output_tput_per_gpu":79.65001,"tput_requests_per_sec":0.65177,"avg_isl":143326.9835,"p50_itl":0.00696,"p90_itl":0.01199,"mean_itl":0.00851,"p50_e2el":3.25027,"p50_tpot":0.00696,"p50_ttft":0.40212,"p75_ttft":0.57798,"p90_e2el":17.6369,"p90_tpot":0.01199,"p90_ttft":0.97136,"mean_e2el":7.72884,"mean_tpot":0.00851,"mean_ttft":0.6369,"stdev_isl":143970.49961,"p50_intvty":143.71275,"p75_intvty":109.79567,"p90_intvty":83.37659,"mean_intvty":117.49398},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":48,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":179785.6728078302,"tput_per_gpu":381.1456263526,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":3228.8293450463,"output_tput_per_gpu":203.1653939343,"p50_itl":1.1304270030232,"mean_itl":"1.0034410328310999","p50_e2el":6.2847666545131,"p50_tpot":0.0047713961568,"p50_ttft":2.056498050486,"mean_e2el":6.3490602587504,"mean_tpot":0.0045585434158,"mean_ttft":"2.1325175706398998","p50_intvty":209.5822621185,"mean_intvty":219.3683176374},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":33,"date":"2026-02-03","num_gpus":26,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":760598.0670237736,"tput_per_gpu":1612.4679020904,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":18618.1736984106,"output_tput_per_gpu":195.3257523971,"p50_itl":1.2316779949761,"mean_itl":1.0714976970518,"p50_e2el":6.2377607244707,"p50_tpot":0.0050761286672,"p50_ttft":1.6779240304895,"mean_e2el":6.1745750056781,"mean_tpot":0.0046900364699,"mean_ttft":1.8366329692255,"p50_intvty":197.0005225557,"mean_intvty":213.2179582007},{"ep":1,"hw":"B300","pp":1,"tp":8,"conc":4,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"2148548.6315789474","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":4082.2424,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":622,"duration_seconds":3602.90508,"input_tput_per_gpu":4054.07326,"output_tput_per_gpu":28.16914,"tput_requests_per_sec":0.17274,"avg_isl":187864.19453,"p50_itl":0.00312,"p90_itl":0.00432,"mean_itl":0.0036,"p50_e2el":"1.8001500000000001","p50_tpot":0.00312,"p50_ttft":"0.36654000000000003","p75_ttft":0.60527,"p90_e2el":13.1565,"p90_tpot":0.00432,"p90_ttft":0.79953,"mean_e2el":"4.8283000000000005","mean_tpot":0.0036,"mean_ttft":0.51281,"stdev_isl":181944.712,"p50_intvty":320.59013,"p75_intvty":265.21742,"p90_intvty":231.74807,"mean_intvty":277.89654},{"ep":1,"hw":"B300","pp":1,"tp":8,"conc":1,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"1450271.2368421054","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":2755.51535,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":279,"duration_seconds":3596.94086,"input_tput_per_gpu":2736.62846,"output_tput_per_gpu":18.88688,"tput_requests_per_sec":0.07739,"avg_isl":282250.63082,"p50_itl":"0.0028599999999999997","p90_itl":0.00344,"mean_itl":0.00299,"p50_e2el":3.28814,"p50_tpot":"0.0028599999999999997","p50_ttft":0.52664,"p75_ttft":0.66343,"p90_e2el":15.71923,"p90_tpot":0.00344,"p90_ttft":0.78873,"mean_e2el":"6.3419300000000005","mean_tpot":0.00299,"mean_ttft":0.65522,"stdev_isl":165048.1481,"p50_intvty":349.177,"p75_intvty":341.08745,"p90_intvty":290.56986,"mean_intvty":333.96069},{"ep":8,"hw":"B300","pp":1,"tp":8,"conc":56,"date":"2026-11-06","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"12988949.173684211","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":24679.00343,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42287685511","total_requests":5509,"duration_seconds":3629.9388,"input_tput_per_gpu":24490.6997,"output_tput_per_gpu":188.30373,"tput_requests_per_sec":1.51764,"avg_isl":129097.46379,"p50_itl":0.0228,"p90_itl":0.02985,"mean_itl":0.02389,"p50_e2el":13.04684,"p50_tpot":0.0228,"p50_ttft":0.9141900000000001,"p75_ttft":2.776,"p90_e2el":55.56483,"p90_tpot":0.02985,"p90_ttft":5.70412,"mean_e2el":25.46584,"mean_tpot":0.02389,"mean_ttft":2.25489,"stdev_isl":142694.81951,"p50_intvty":43.85237,"p75_intvty":39.06575,"p90_intvty":33.502,"mean_intvty":41.86331},{"ep":32,"hw":"GB300 NVL72","tp":40,"conc":384,"date":"2026-11-08","num_gpus":56,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"14344240.669811321","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":30409.79022,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":46196,"duration_seconds":3629.55145,"input_tput_per_gpu":30181.34371,"output_tput_per_gpu":228.44651,"tput_requests_per_sec":12.7294,"avg_isl":132793.0,"p50_itl":0.01302,"p90_itl":"0.013869999999999999","mean_itl":0.01298,"p50_e2el":9.43652,"p50_tpot":0.01302,"p50_ttft":"2.2999099999999997","p75_ttft":4.28813,"p90_e2el":34.89559,"p90_tpot":"0.013869999999999999","p90_ttft":8.07976,"mean_e2el":16.55209,"mean_tpot":0.01298,"mean_ttft":"3.6817699999999998","stdev_isl":134628.85055,"p50_intvty":76.78739,"p75_intvty":74.76437,"p90_intvty":72.10666,"mean_intvty":77.02674},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":32,"date":"2026-11-08","num_gpus":32,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"3558881.1886792453","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":7544.82812,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":4792,"duration_seconds":3612.07639,"input_tput_per_gpu":7501.52597,"output_tput_per_gpu":43.30215,"tput_requests_per_sec":1.32678,"avg_isl":180942.13585,"p50_itl":0.00362,"p90_itl":0.004730000000000001,"mean_itl":0.0037,"p50_e2el":2.42061,"p50_tpot":0.00362,"p50_ttft":0.6443300000000001,"p75_ttft":0.92367,"p90_e2el":10.50818,"p90_tpot":0.004730000000000001,"p90_ttft":1.47524,"mean_e2el":4.69221,"mean_tpot":0.0037,"mean_ttft":0.90907,"stdev_isl":161763.9562,"p50_intvty":276.42083,"p75_intvty":243.30279,"p90_intvty":211.32452,"mean_intvty":270.20139},{"ep":32,"hw":"GB300 NVL72","tp":40,"conc":192,"date":"2026-11-08","num_gpus":48,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":9657684.688679244,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":20474.29154,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":25177,"duration_seconds":3629.25751,"input_tput_per_gpu":20326.20569,"output_tput_per_gpu":148.08586,"tput_requests_per_sec":6.93881,"avg_isl":140640.80943,"p50_itl":0.01,"p90_itl":0.01067,"mean_itl":0.01,"p50_e2el":6.213439999999999,"p50_tpot":0.01,"p50_ttft":"1.4464000000000001","p75_ttft":2.25225,"p90_e2el":26.52068,"p90_tpot":0.01067,"p90_ttft":3.49927,"mean_e2el":12.19285,"mean_tpot":0.01,"mean_ttft":2.03811,"stdev_isl":139008.34425,"p50_intvty":100.0001,"p75_intvty":97.10338,"p90_intvty":93.73173,"mean_intvty":99.97714},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":20,"date":"2026-11-08","num_gpus":32,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"2173758.6226415094","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":4608.36828,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":3168,"duration_seconds":3603.29918,"input_tput_per_gpu":4579.60457,"output_tput_per_gpu":28.76371,"tput_requests_per_sec":0.87899,"avg_isl":166683.69066,"p50_itl":0.00311,"p90_itl":0.00379,"mean_itl":0.00318,"p50_e2el":1.99301,"p50_tpot":0.00311,"p50_ttft":0.56945,"p75_ttft":0.81036,"p90_e2el":9.24311,"p90_tpot":0.00379,"p90_ttft":1.17967,"mean_e2el":"4.0020999999999995","mean_tpot":0.00318,"mean_ttft":0.7159800000000001,"stdev_isl":148933.53257,"p50_intvty":321.57193,"p75_intvty":283.98507,"p90_intvty":264.09433,"mean_intvty":314.43574},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":56,"date":"2026-11-08","num_gpus":32,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":5829840.339622642,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":12359.26152,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":9384,"duration_seconds":3612.00504,"input_tput_per_gpu":12276.20623,"output_tput_per_gpu":83.05528,"tput_requests_per_sec":2.59845,"avg_isl":151207.9074,"p50_itl":0.00509,"p90_itl":0.00625,"mean_itl":0.00514,"p50_e2el":3.41309,"p50_tpot":0.00509,"p50_ttft":0.8115,"p75_ttft":"1.3186900000000001","p90_e2el":14.1564,"p90_tpot":0.00625,"p90_ttft":2.20342,"mean_e2el":6.43799,"mean_tpot":0.00514,"mean_ttft":1.29691,"stdev_isl":167756.86053,"p50_intvty":196.5244,"p75_intvty":175.29217,"p90_intvty":160.01049,"mean_intvty":194.39423},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":57577.64638075472,"tput_per_gpu":122.0646103272,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":1033.7031177888,"output_tput_per_gpu":65.0872036109,"p50_itl":0.8192278609495001,"mean_itl":0.7204120387978,"p50_e2el":3.5515489155077,"p50_tpot":0.0035179187086,"p50_ttft":0.289771859505,"mean_e2el":3.4310676972855,"mean_tpot":0.00332958796,"mean_ttft":0.3423012341402,"p50_intvty":284.2589845979,"mean_intvty":300.3374627808},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-23","num_gpus":72,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"471302.08435043803","tput_per_gpu":645.6838555601,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":64,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2904.2384548479,"output_tput_per_gpu":363.3645306491,"p50_itl":4.2018386045238,"mean_itl":3.9501280332956,"p50_e2el":"18.861431284924002","p50_tpot":0.0183580459615,"p50_ttft":1.3376573454589,"mean_e2el":19.2395805270597,"mean_tpot":0.0184382459675,"mean_ttft":2.2648312946147,"p50_intvty":54.4720283465,"mean_intvty":54.2350938241},{"ep":4,"hw":"GB300 NVL72","tp":4,"conc":46,"date":"2026-11-03","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"30012161.759433962","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":63625.78293,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":5852,"duration_seconds":3627.2767,"input_tput_per_gpu":63204.81883,"output_tput_per_gpu":420.96409,"tput_requests_per_sec":1.6163,"avg_isl":156706.3339,"p50_itl":"0.0077599999999999995","p90_itl":0.009859999999999999,"mean_itl":0.00767,"p50_e2el":5.43372,"p50_tpot":"0.0077599999999999995","p50_ttft":0.98493,"p75_ttft":1.81302,"p90_e2el":"24.941740000000003","p90_tpot":0.009859999999999999,"p90_ttft":5.21069,"mean_e2el":10.85612,"mean_tpot":0.00767,"mean_ttft":2.28131,"stdev_isl":162837.43677,"p50_intvty":128.91856,"p75_intvty":111.88822,"p90_intvty":101.44731,"mean_intvty":130.32292},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":352,"date":"2026-11-03","num_gpus":40,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":23113366.29245283,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":49000.33654,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":51494,"duration_seconds":3628.70731,"input_tput_per_gpu":48636.72954,"output_tput_per_gpu":363.607,"tput_requests_per_sec":14.22431,"avg_isl":137094.38451,"p50_itl":0.0057,"p90_itl":0.00643,"mean_itl":0.00558,"p50_e2el":5.19398,"p50_tpot":0.0057,"p50_ttft":2.32594,"p75_ttft":"3.1053200000000003","p90_e2el":18.11302,"p90_tpot":0.00643,"p90_ttft":4.371020000000001,"mean_e2el":9.11174,"mean_tpot":0.00558,"mean_ttft":3.12617,"stdev_isl":135052.80082,"p50_intvty":175.32934,"p75_intvty":162.32858,"p90_intvty":155.4401,"mean_intvty":179.15842},{"ep":16,"hw":"GB300 NVL72","tp":32,"conc":352,"date":"2026-11-08","num_gpus":32,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"17403030.235849056","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":36894.4241,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":34546,"duration_seconds":3627.71129,"input_tput_per_gpu":36608.18965,"output_tput_per_gpu":286.23445,"tput_requests_per_sec":9.5244,"avg_isl":123016.44677,"p50_itl":"0.020030000000000003","p90_itl":0.02079,"mean_itl":0.01977,"p50_e2el":14.52332,"p50_tpot":"0.020030000000000003","p50_ttft":4.21866,"p75_ttft":6.167479999999999,"p90_e2el":56.35976,"p90_tpot":0.02079,"p90_ttft":11.37865,"mean_e2el":26.83152,"mean_tpot":0.01977,"mean_ttft":"7.9144499999999995","stdev_isl":128046.37286,"p50_intvty":49.93585,"p75_intvty":48.94979,"p90_intvty":48.10902,"mean_intvty":50.57251},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":112,"date":"2026-11-03","num_gpus":40,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":8862825.882075472,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":18789.19087,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":19235,"duration_seconds":3613.72837,"input_tput_per_gpu":18650.01187,"output_tput_per_gpu":139.17899,"tput_requests_per_sec":5.33537,"avg_isl":140153.00639,"p50_itl":0.00289,"p90_itl":0.00326,"mean_itl":0.00282,"p50_e2el":3.05849,"p50_tpot":0.00289,"p50_ttft":0.9719800000000001,"p75_ttft":2.27903,"p90_e2el":10.4879,"p90_tpot":0.00326,"p90_ttft":4.44158,"mean_e2el":4.89838,"mean_tpot":0.00282,"mean_ttft":"1.8363399999999999","stdev_isl":147691.28383,"p50_intvty":346.34305,"p75_intvty":323.96743,"p90_intvty":306.82724,"mean_intvty":354.17012},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":672,"date":"2026-11-03","num_gpus":56,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"25458527.811320756","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":53972.07896,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":86775,"duration_seconds":3629.67404,"input_tput_per_gpu":53547.18764,"output_tput_per_gpu":424.89131,"tput_requests_per_sec":23.95748,"avg_isl":125428.92372,"p50_itl":0.00644,"p90_itl":0.00735,"mean_itl":0.00631,"p50_e2el":5.57932,"p50_tpot":0.00644,"p50_ttft":2.4091,"p75_ttft":3.13665,"p90_e2el":19.64538,"p90_tpot":0.00735,"p90_ttft":4.44696,"mean_e2el":10.99639,"mean_tpot":0.00631,"mean_ttft":4.398020000000001,"stdev_isl":110767.4765,"p50_intvty":155.21663,"p75_intvty":143.70315,"p90_intvty":136.06132,"mean_intvty":158.42422},{"ep":4,"hw":"GB300 NVL72","tp":4,"conc":56,"date":"2026-11-03","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":33166805.74528302,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":70313.62818,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":7266,"duration_seconds":3626.79301,"input_tput_per_gpu":69800.95542,"output_tput_per_gpu":512.67277,"tput_requests_per_sec":2.00691,"avg_isl":139363.4005,"p50_itl":0.01051,"p90_itl":"0.013519999999999999","mean_itl":0.0104,"p50_e2el":7.52838,"p50_tpot":0.01051,"p50_ttft":1.14518,"p75_ttft":"3.2365500000000003","p90_e2el":32.00854,"p90_tpot":"0.013519999999999999","p90_ttft":6.667470000000001,"mean_e2el":14.23533,"mean_tpot":0.0104,"mean_ttft":2.79108,"stdev_isl":154213.9544,"p50_intvty":95.15224,"p75_intvty":84.00648,"p90_intvty":73.94004,"mean_intvty":96.17273},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":120,"date":"2026-11-03","num_gpus":48,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":7691755.202830189,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":16306.52103,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":20136,"duration_seconds":3620.59019,"input_tput_per_gpu":16183.32282,"output_tput_per_gpu":123.19821,"tput_requests_per_sec":5.57293,"avg_isl":139673.84942,"p50_itl":0.00265,"p90_itl":0.00299,"mean_itl":0.0026,"p50_e2el":"2.7082800000000002","p50_tpot":0.00265,"p50_ttft":0.8292999999999999,"p75_ttft":1.9543,"p90_e2el":10.05491,"p90_tpot":0.00299,"p90_ttft":4.18215,"mean_e2el":4.57346,"mean_tpot":0.0026,"mean_ttft":1.70983,"stdev_isl":146519.43731,"p50_intvty":377.80497,"p75_intvty":350.28465,"p90_intvty":334.37544,"mean_intvty":384.1964},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":1,"date":"2026-11-08","num_gpus":16,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":655497.4386792453,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":1389.65457,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":280,"duration_seconds":3591.71282,"input_tput_per_gpu":1380.18456,"output_tput_per_gpu":9.47001,"tput_requests_per_sec":0.07778,"avg_isl":283270.08929,"p50_itl":0.00272,"p90_itl":0.0029,"mean_itl":0.00276,"p50_e2el":"3.1691599999999998","p50_tpot":0.00272,"p50_ttft":0.76202,"p75_ttft":"1.0811099999999998","p90_e2el":15.35972,"p90_tpot":0.0029,"p90_ttft":1.27748,"mean_e2el":6.14592,"mean_tpot":0.00276,"mean_ttft":0.8065800000000001,"stdev_isl":165628.85185,"p50_intvty":367.57043,"p75_intvty":359.02843,"p90_intvty":344.56748,"mean_intvty":361.66544},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":48,"date":"2026-11-08","num_gpus":24,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":6204236.976415094,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":13152.98239,"model_display":"Kimi-K3.1-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794124800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42501370812","total_requests":7251,"duration_seconds":3623.24705,"input_tput_per_gpu":13066.74216,"output_tput_per_gpu":86.24023,"tput_requests_per_sec":2.00166,"avg_isl":156703.46683,"p50_itl":0.00591,"p90_itl":"0.0070999999999999995","mean_itl":0.00584,"p50_e2el":3.56602,"p50_tpot":0.00591,"p50_ttft":0.73527,"p75_ttft":1.13542,"p90_e2el":15.3339,"p90_tpot":"0.0070999999999999995","p90_ttft":1.86259,"mean_e2el":7.00123,"mean_tpot":0.00584,"mean_ttft":1.08392,"stdev_isl":170175.91627,"p50_intvty":169.13926,"p75_intvty":152.75112,"p90_intvty":140.92009,"mean_intvty":171.33855},{"ep":4,"hw":"GB300 NVL72","tp":4,"conc":50,"date":"2026-11-03","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"31270030.018867925","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":66292.46364,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":6534,"duration_seconds":3629.18718,"input_tput_per_gpu":65831.04758,"output_tput_per_gpu":461.41606,"tput_requests_per_sec":1.8037,"avg_isl":146258.4596,"p50_itl":0.00886,"p90_itl":0.01088,"mean_itl":0.00862,"p50_e2el":6.10205,"p50_tpot":0.00886,"p50_ttft":0.99622,"p75_ttft":"2.0420599999999998","p90_e2el":27.19696,"p90_tpot":0.01088,"p90_ttft":6.167,"mean_e2el":"11.890049999999999","mean_tpot":0.00862,"mean_ttft":"2.4464200000000003","stdev_isl":155913.52216,"p50_intvty":112.91366,"p75_intvty":100.29077,"p90_intvty":91.89956,"mean_intvty":116.0351},{"ep":32,"hw":"Vera Rubin NVL72","tp":32,"conc":1229,"date":"2026-10-05","num_gpus":48,"scenario":"agentic","decode_ep":32,"decode_tp":32,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":41679510.3969697,"tput_per_gpu":137542.38431,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":32,"serving_config":"disagg","total_requests":168229,"num_prefill_gpu":16,"duration_seconds":3629.12782,"input_tput_per_gpu":409634.33232,"total_input_tokens":23793566998,"output_tput_per_gpu":1496.410305,"total_output_tokens":174097308,"avg_output_tokens_expected":1356.24147,"theoretical_cache_hit_rate":0.97096,"stdev_output_tokens_expected":2537.39624,"avg_isl":141389.68669,"avg_osl":1033.00416,"p50_itl":0.00491,"p75_isl":179124.0,"p75_osl":1032.0,"p75_qps":54.0,"p90_isl":307931.2,"p90_itl":0.00536,"p90_osl":2416.0,"p90_qps":60.0,"p95_isl":445835.0,"p95_osl":3963.0,"p95_qps":64.0,"std_qps":11.54729,"mean_itl":"0.0050999999999999995","mean_qps":46.36907,"p50_e2el":7.114439999999999,"p50_tpot":0.004889999999999999,"p50_ttft":4.02537,"p90_e2el":21.84798,"p90_tpot":0.00523,"p90_ttft":10.43738,"mean_e2el":10.61668,"mean_tpot":"0.0049900000000000005","mean_ttft":5.58859,"stdev_isl":139796.20753,"stdev_osl":2009.9222,"median_qps":47.0,"p50_intvty":203.66598778,"p90_intvty":186.5671641791,"mean_intvty":196.0784313725},{"ep":16,"hw":"Vera Rubin NVL72","tp":16,"conc":2934,"date":"2026-10-05","num_gpus":46,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":6,"prefill_tp":6,"tput_per_mw":85076826.92121212,"tput_per_gpu":280753.52884,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":338941,"num_prefill_gpu":30,"duration_seconds":3629.88653,"input_tput_per_gpu":427324.996916,"total_input_tokens":46585614014,"output_tput_per_gpu":5932.02616875,"total_output_tokens":345909594,"avg_output_tokens_expected":1349.42158,"theoretical_cache_hit_rate":0.96968,"stdev_output_tokens_expected":2539.42961,"avg_isl":137293.0319,"avg_osl":1016.46396,"p50_itl":0.00967,"p75_isl":170770.0,"p75_osl":1017.0,"p75_qps":106.0,"p90_isl":296103.0,"p90_itl":0.0103,"p90_osl":2372.0,"p90_qps":115.0,"p95_isl":436015.0,"p95_osl":3893.0,"p95_qps":122.0,"std_qps":19.39549,"mean_itl":0.00983,"mean_qps":93.39598,"p50_e2el":10.86143,"p50_tpot":0.009640000000000001,"p50_ttft":4.538189999999999,"p90_e2el":36.75049,"p90_tpot":0.01016,"p90_ttft":17.10651,"mean_e2el":17.15205,"mean_tpot":0.009710000000000002,"mean_ttft":7.502689999999999,"stdev_isl":137603.59133,"stdev_osl":1981.70018,"median_qps":95.0,"p50_intvty":103.4126163392,"p90_intvty":97.0873786408,"mean_intvty":101.7293997965},{"ep":32,"hw":"Vera Rubin NVL72","tp":32,"conc":368,"date":"2026-10-05","num_gpus":38,"scenario":"agentic","decode_ep":32,"decode_tp":32,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":6,"prefill_tp":6,"tput_per_mw":"14662177.863636365","tput_per_gpu":48385.18695,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":32,"serving_config":"disagg","total_requests":47949,"num_prefill_gpu":6,"duration_seconds":3629.92091,"input_tput_per_gpu":304155.4970133333,"total_input_tokens":6628971190,"output_tput_per_gpu":428.253813125,"total_output_tokens":49875447,"avg_output_tokens_expected":1360.87099,"theoretical_cache_hit_rate":0.97008,"stdev_output_tokens_expected":2544.92908,"avg_isl":138154.33882,"avg_osl":1037.45394,"p50_itl":"0.0044599999999999996","p75_isl":173583.0,"p75_osl":1048.0,"p75_qps":16.0,"p90_isl":298294.2,"p90_itl":0.00466,"p90_osl":2423.2,"p90_qps":19.0,"p95_isl":444072.8,"p95_osl":3953.6,"p95_qps":20.0,"std_qps":4.08821,"mean_itl":0.00458,"mean_qps":13.2161,"p50_e2el":11.15878,"p50_tpot":"0.0044599999999999996","p50_ttft":8.36618,"p90_e2el":22.129,"p90_tpot":"0.0046500000000000005","p90_ttft":12.85517,"mean_e2el":13.37284,"mean_tpot":0.00457,"mean_ttft":8.786700000000002,"stdev_isl":140955.32349,"stdev_osl":1979.81466,"median_qps":13.0,"p50_intvty":224.2152466368,"p90_intvty":214.5922746781,"mean_intvty":218.3406113537},{"ep":32,"hw":"Vera Rubin NVL72","tp":32,"conc":2087,"date":"2026-10-05","num_gpus":56,"scenario":"agentic","decode_ep":32,"decode_tp":32,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":6,"prefill_tp":6,"tput_per_mw":59153954.13333333,"tput_per_gpu":195208.04864,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":32,"serving_config":"disagg","total_requests":279170,"num_prefill_gpu":24,"duration_seconds":3629.47278,"input_tput_per_gpu":452181.9592266667,"total_input_tokens":39402135064,"output_tput_per_gpu":2477.6157,"total_output_tokens":288460617,"avg_output_tokens_expected":1359.60853,"theoretical_cache_hit_rate":0.97086,"stdev_output_tokens_expected":2540.61401,"avg_isl":141090.98668,"avg_osl":1030.76276,"p50_itl":0.005679999999999999,"p75_isl":178317.0,"p75_osl":1025.0,"p75_qps":89.0,"p90_isl":304314.5,"p90_itl":0.00605,"p90_osl":2408.0,"p90_qps":96.0,"p95_isl":440032.75,"p95_osl":3955.0,"p95_qps":101.6,"std_qps":17.15888,"mean_itl":0.00584,"mean_qps":76.92725,"p50_e2el":7.12074,"p50_tpot":0.00567,"p50_ttft":3.05868,"p90_e2el":25.03674,"p90_tpot":0.00597,"p90_ttft":12.92072,"mean_e2el":11.47297,"mean_tpot":0.00578,"mean_ttft":5.6712,"stdev_isl":139176.5592,"stdev_osl":2008.31513,"median_qps":79.0,"p50_intvty":176.0563380282,"p90_intvty":165.2892561983,"mean_intvty":171.2328767123},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":48,"date":"2026-10-05","num_gpus":28,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"3636870.5212121215","tput_per_gpu":12001.67272,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":24,"serving_config":"disagg","total_requests":7389,"num_prefill_gpu":4,"duration_seconds":3618.13362,"input_tput_per_gpu":83467.41725,"total_input_tokens":1209019139,"output_tput_per_gpu":90.7152866667,"total_output_tokens":7877285,"avg_output_tokens_expected":1295.12816,"theoretical_cache_hit_rate":0.9764,"stdev_output_tokens_expected":2136.14098,"avg_isl":163484.24361,"avg_osl":1066.08215,"p50_itl":0.00377,"p75_isl":210933.0,"p75_osl":1084.0,"p75_qps":3.0,"p90_isl":369210.2,"p90_itl":0.00435,"p90_osl":2587.0,"p90_qps":4.0,"p95_isl":609729.4,"p95_osl":4161.2,"p95_qps":5.0,"std_qps":1.55531,"mean_itl":0.00385,"mean_qps":2.04258,"p50_e2el":2.2537,"p50_tpot":0.00376,"p50_ttft":"0.39518000000000003","p90_e2el":10.58005,"p90_tpot":0.00435,"p90_ttft":0.97677,"mean_e2el":4.6452,"mean_tpot":"0.0038399999999999997","mean_ttft":0.6688,"stdev_isl":172808.06158,"stdev_osl":2140.86625,"median_qps":2.0,"p50_intvty":265.2519893899,"p90_intvty":229.8850574713,"mean_intvty":259.7402597403},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":6,"date":"2026-10-05","num_gpus":28,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":419821.5363636364,"tput_per_gpu":1385.41107,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":24,"serving_config":"disagg","total_requests":864,"num_prefill_gpu":4,"duration_seconds":3602.98627,"input_tput_per_gpu":9623.94328,"total_input_tokens":139210803,"output_tput_per_gpu":12.3223683333,"total_output_tokens":1065538,"avg_output_tokens_expected":1747.29737,"theoretical_cache_hit_rate":0.97738,"stdev_output_tokens_expected":2716.22965,"avg_isl":160532.1088,"avg_osl":1233.25926,"p50_itl":0.00315,"p75_isl":215130.75,"p75_osl":1314.0,"p75_qps":0.0,"p90_isl":442877.9,"p90_itl":0.00323,"p90_osl":3064.5,"p90_qps":1.0,"p95_isl":537932.7,"p95_osl":4564.35,"p95_qps":1.0,"std_qps":0.50924,"mean_itl":0.0032,"mean_qps":0.24289,"p50_e2el":2.26625,"p50_tpot":0.00315,"p50_ttft":0.32151,"p90_e2el":"11.844790000000001","p90_tpot":0.00323,"p90_ttft":0.96447,"mean_e2el":4.64596,"mean_tpot":0.00319,"mean_ttft":0.7459199999999999,"stdev_isl":154394.73715,"stdev_osl":2062.84297,"median_qps":0.0,"p50_intvty":317.4603174603,"p90_intvty":309.5975232198,"mean_intvty":312.5},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":160,"date":"2026-11-03","num_gpus":24,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"19375447.429245286","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":41075.94855,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":24168,"duration_seconds":3628.2568,"input_tput_per_gpu":40779.59108,"output_tput_per_gpu":296.35747,"tput_requests_per_sec":6.67597,"avg_isl":146930.31649,"p50_itl":0.00451,"p90_itl":"0.0049900000000000005","mean_itl":0.00442,"p50_e2el":3.92281,"p50_tpot":0.00451,"p50_ttft":1.51732,"p75_ttft":2.46152,"p90_e2el":14.24284,"p90_tpot":"0.0049900000000000005","p90_ttft":"3.6625799999999997","mean_e2el":"6.9276599999999995","mean_tpot":0.00442,"mean_ttft":2.00756,"stdev_isl":145149.8807,"p50_intvty":221.53375,"p75_intvty":208.97327,"p90_intvty":200.466,"mean_intvty":226.06448},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":224,"date":"2026-11-03","num_gpus":32,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"15935385.783018868","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":33783.01786,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":30356,"duration_seconds":3627.26388,"input_tput_per_gpu":33520.27298,"output_tput_per_gpu":262.74488,"tput_requests_per_sec":8.38768,"avg_isl":128171.69647,"p50_itl":0.00393,"p90_itl":0.00445,"mean_itl":0.00396,"p50_e2el":4.45628,"p50_tpot":0.00393,"p50_ttft":2.42698,"p75_ttft":"3.2759699999999996","p90_e2el":13.52917,"p90_tpot":0.00445,"p90_ttft":4.44426,"mean_e2el":8.23682,"mean_tpot":0.00396,"mean_ttft":4.19451,"stdev_isl":102781.07916,"p50_intvty":254.29066,"p75_intvty":237.70842,"p90_intvty":224.72916,"mean_intvty":252.5379},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":32,"date":"2026-11-03","num_gpus":40,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":"3090150.8679245287","dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":6551.11984,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":5073,"duration_seconds":3602.08818,"input_tput_per_gpu":6512.76685,"output_tput_per_gpu":38.35298,"tput_requests_per_sec":1.41163,"avg_isl":184975.83698,"p50_itl":0.00188,"p90_itl":"0.0022299999999999998","mean_itl":0.00191,"p50_e2el":1.2658,"p50_tpot":0.00188,"p50_ttft":0.24114,"p75_ttft":0.40194,"p90_e2el":6.21557,"p90_tpot":"0.0022299999999999998","p90_ttft":0.91773,"mean_e2el":2.61769,"mean_tpot":0.00191,"mean_ttft":"0.43454000000000004","stdev_isl":165033.07423,"p50_intvty":530.93,"p75_intvty":487.84381,"p90_intvty":448.45775,"mean_intvty":523.54937},{"ep":8,"hw":"GB300 NVL72","tp":16,"conc":56,"date":"2026-11-03","num_gpus":40,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","dataset_url":"https://huggingface.co/datasets/semianalysisai/cc-traces-weka-062126","tput_per_mw":5334148.646226415,"dataset_name":"semianalysisai/cc-traces-weka-062126","tput_per_gpu":11308.39513,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41611203638","total_requests":10325,"duration_seconds":3602.41484,"input_tput_per_gpu":11233.56658,"output_tput_per_gpu":74.82856,"tput_requests_per_sec":2.87229,"avg_isl":156776.62731,"p50_itl":0.00222,"p90_itl":0.00248,"mean_itl":0.00218,"p50_e2el":"1.8635599999999999","p50_tpot":0.00222,"p50_ttft":0.40539,"p75_ttft":1.03403,"p90_e2el":7.4684,"p90_tpot":0.00248,"p90_ttft":"2.2210199999999998","mean_e2el":3.28534,"mean_tpot":0.00218,"mean_ttft":0.9223,"stdev_isl":171401.71721,"p50_intvty":451.19302,"p75_intvty":420.38577,"p90_intvty":402.54784,"mean_intvty":458.38076},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":160,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":18207842.08018868,"tput_per_gpu":38600.62521,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":23208,"num_prefill_gpu":8,"duration_seconds":3628.76019,"input_tput_per_gpu":114956.40762,"total_input_tokens":6676806714,"output_tput_per_gpu":422.734005,"total_output_tokens":24654608,"avg_output_tokens_expected":1339.56768,"theoretical_cache_hit_rate":0.97216,"stdev_output_tokens_expected":2505.5814,"avg_isl":143794.97962,"avg_osl":1057.56657,"p50_itl":0.006019999999999999,"p75_isl":183578.75,"p75_osl":1016.0,"p75_qps":8.0,"p90_isl":302704.1,"p90_itl":0.00652,"p90_osl":2449.9,"p90_qps":11.0,"p95_isl":444694.25,"p95_osl":4054.9,"p95_qps":12.0,"std_qps":3.27654,"mean_itl":0.00637,"mean_qps":6.40342,"p50_e2el":4.93337,"p50_tpot":0.00601,"p50_ttft":"1.7774100000000002","p90_e2el":17.53584,"p90_tpot":0.0065,"p90_ttft":4.11364,"mean_e2el":8.60041,"mean_tpot":0.00636,"mean_ttft":2.31627,"stdev_isl":142931.504,"stdev_osl":2113.60231,"median_qps":6.0,"p50_intvty":166.1129568106,"p90_intvty":153.3742331288,"mean_intvty":156.9858712716},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":300,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"31991827.372641508","tput_per_gpu":67822.67403,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":42439,"num_prefill_gpu":8,"duration_seconds":3629.68755,"input_tput_per_gpu":201979.33821,"total_input_tokens":11733649612,"output_tput_per_gpu":744.341925,"total_output_tokens":43327191,"avg_output_tokens_expected":1327.53498,"theoretical_cache_hit_rate":0.97096,"stdev_output_tokens_expected":2390.33388,"avg_isl":138197.76908,"avg_osl":1018.58333,"p50_itl":0.00684,"p75_isl":174674.5,"p75_osl":1021.0,"p75_qps":15.0,"p90_isl":293491.4,"p90_itl":0.00735,"p90_osl":2355.4,"p90_qps":18.0,"p95_isl":425210.5,"p95_osl":3852.1,"p95_qps":20.0,"std_qps":5.23139,"mean_itl":"0.0072699999999999996","mean_qps":11.7359,"p50_e2el":"7.0831800000000005","p50_tpot":0.00683,"p50_ttft":2.83837,"p90_e2el":21.41404,"p90_tpot":0.00732,"p90_ttft":7.79959,"mean_e2el":"10.922379999999999","mean_tpot":0.00726,"mean_ttft":4.06639,"stdev_isl":136706.67869,"stdev_osl":2035.45866,"median_qps":12.0,"p50_intvty":146.1988304094,"p90_intvty":136.0544217687,"mean_intvty":137.5515818432},{"ep":12,"hw":"GB300 NVL72","tp":12,"conc":800,"date":"2026-10-23","num_gpus":28,"scenario":"agentic","decode_ep":12,"decode_tp":12,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"60716021.405660376","tput_per_gpu":128717.96538,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":12,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":96115,"num_prefill_gpu":16,"duration_seconds":3629.76685,"input_tput_per_gpu":223588.8411575,"total_input_tokens":25993955548,"output_tput_per_gpu":2223.4643433333,"total_output_tokens":97310370,"avg_output_tokens_expected":1358.54115,"theoretical_cache_hit_rate":0.96993,"stdev_output_tokens_expected":2564.23725,"avg_isl":135100.72117,"avg_osl":1007.62509,"p50_itl":0.01126,"p75_isl":169528.5,"p75_osl":1011.0,"p75_qps":32.0,"p90_isl":287554.8,"p90_itl":"0.012539999999999999","p90_osl":2336.0,"p90_qps":38.0,"p95_isl":424607.5,"p95_osl":3851.3,"p95_qps":42.0,"std_qps":10.73541,"mean_itl":"0.011619999999999998","mean_qps":26.5435,"p50_e2el":10.75307,"p50_tpot":0.01122,"p50_ttft":4.1535,"p90_e2el":33.04261,"p90_tpot":0.01239,"p90_ttft":11.25096,"mean_e2el":16.76172,"mean_tpot":"0.011519999999999999","mean_ttft":5.72103,"stdev_isl":134917.59546,"stdev_osl":1954.95863,"median_qps":26.0,"p50_intvty":88.809946714,"p90_intvty":79.7448165869,"mean_intvty":86.0585197935},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":200,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"22449739.273584906","tput_per_gpu":47593.44726,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":28380,"num_prefill_gpu":8,"duration_seconds":3624.51746,"input_tput_per_gpu":141757.9566,"total_input_tokens":8224852640,"output_tput_per_gpu":511.192605,"total_output_tokens":29811306,"avg_output_tokens_expected":1352.13258,"theoretical_cache_hit_rate":0.97147,"stdev_output_tokens_expected":2420.48468,"avg_isl":144835.57118,"avg_osl":1044.58154,"p50_itl":0.00628,"p75_isl":179468.0,"p75_osl":999.0,"p75_qps":10.0,"p90_isl":302712.3,"p90_itl":"0.0068200000000000005","p90_osl":2430.1,"p90_qps":13.0,"p95_isl":449481.3,"p95_osl":4033.25,"p95_qps":15.0,"std_qps":3.83501,"mean_itl":0.006690000000000001,"mean_qps":7.84817,"p50_e2el":5.56813,"p50_tpot":0.00628,"p50_ttft":"2.0920199999999998","p90_e2el":"18.800939999999997","p90_tpot":0.0068,"p90_ttft":5.392810000000001,"mean_e2el":9.28241,"mean_tpot":0.00667,"mean_ttft":2.80447,"stdev_isl":142055.48681,"stdev_osl":2130.54422,"median_qps":8.0,"p50_intvty":159.2356687898,"p90_intvty":146.6275659824,"mean_intvty":149.4768310912},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":80,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"10328144.735849058","tput_per_gpu":21895.66684,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":13882,"num_prefill_gpu":8,"duration_seconds":3618.49796,"input_tput_per_gpu":65181.86142,"total_input_tokens":3782510766,"output_tput_per_gpu":252.569565,"total_output_tokens":14662473,"avg_output_tokens_expected":1344.007,"theoretical_cache_hit_rate":0.96996,"stdev_output_tokens_expected":2501.76227,"avg_isl":135923.02694,"avg_osl":1053.36112,"p50_itl":0.00549,"p75_isl":170727.75,"p75_osl":1099.75,"p75_qps":5.0,"p90_isl":301594.8,"p90_itl":0.00595,"p90_osl":2380.9,"p90_qps":7.0,"p95_isl":498108.85,"p95_osl":3987.95,"p95_qps":8.0,"std_qps":2.36591,"mean_itl":0.00588,"mean_qps":3.8409,"p50_e2el":4.30471,"p50_tpot":0.00549,"p50_ttft":1.20786,"p90_e2el":15.29782,"p90_tpot":0.00594,"p90_ttft":3.0754,"mean_e2el":7.429270000000001,"mean_tpot":0.00587,"mean_ttft":1.76316,"stdev_isl":156544.69269,"stdev_osl":1976.67165,"median_qps":4.0,"p50_intvty":182.1493624772,"p90_intvty":168.0672268908,"mean_intvty":170.0680272109},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":720,"date":"2026-10-23","num_gpus":32,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":52642945.08962265,"tput_per_gpu":111603.04359,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":93443,"num_prefill_gpu":16,"duration_seconds":3629.60315,"input_tput_per_gpu":221580.5127,"total_input_tokens":25753180196,"output_tput_per_gpu":1625.5745,"total_output_tokens":94889326,"avg_output_tokens_expected":1356.65063,"theoretical_cache_hit_rate":0.97047,"stdev_output_tokens_expected":2526.29443,"avg_isl":137709.50455,"avg_osl":1010.27412,"p50_itl":0.00891,"p75_isl":172919.0,"p75_osl":1014.0,"p75_qps":31.0,"p90_isl":296430.0,"p90_itl":0.01001,"p90_osl":2350.0,"p90_qps":37.0,"p95_isl":437352.8,"p95_osl":3860.0,"p95_qps":41.0,"std_qps":10.33755,"mean_itl":0.00937,"mean_qps":25.79078,"p50_e2el":9.07776,"p50_tpot":0.00887,"p50_ttft":3.7264,"p90_e2el":26.99914,"p90_tpot":0.009890000000000001,"p90_ttft":9.6707,"mean_e2el":13.84585,"mean_tpot":0.00929,"mean_ttft":5.03162,"stdev_isl":138853.66379,"stdev_osl":1939.03407,"median_qps":26.0,"p50_intvty":112.2334455668,"p90_intvty":99.9000999001,"mean_intvty":106.7235859125},{"ep":8,"hw":"Vera Rubin NVL72","tp":8,"conc":1882,"date":"2026-10-05","num_gpus":26,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":6,"prefill_tp":6,"tput_per_mw":86478760.21212122,"tput_per_gpu":285379.9087,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1791183600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":198295,"num_prefill_gpu":18,"duration_seconds":3629.9906,"input_tput_per_gpu":409164.5520011111,"total_input_tokens":26795704250,"output_tput_per_gpu":6864.461305,"total_output_tokens":200950401,"avg_output_tokens_expected":1336.34258,"theoretical_cache_hit_rate":0.96896,"stdev_output_tokens_expected":2526.2679,"avg_isl":134823.07982,"avg_osl":1005.28727,"p50_itl":0.01208,"p75_isl":167010.0,"p75_osl":1011.5,"p75_qps":63.0,"p90_isl":292530.6,"p90_itl":0.01248,"p90_osl":2341.0,"p90_qps":70.0,"p95_isl":433465.6,"p95_osl":3849.0,"p95_qps":73.0,"std_qps":12.42162,"mean_itl":0.01209,"mean_qps":54.63985,"p50_e2el":"13.759879999999999","p50_tpot":0.012070000000000001,"p50_ttft":5.68183,"p90_e2el":"44.853519999999996","p90_tpot":0.01244,"p90_ttft":22.38614,"mean_e2el":21.2937,"mean_tpot":"0.012060000000000001","mean_ttft":9.412889999999999,"stdev_isl":135316.23735,"stdev_osl":1962.82757,"median_qps":55.0,"p50_intvty":82.7814569536,"p90_intvty":80.1282051282,"mean_intvty":82.7129859388},{"ep":1,"hw":"GB300 NVL72","tp":4,"conc":1,"date":"2026-10-23","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":2283375.231132075,"tput_per_gpu":4840.75549,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":263,"duration_seconds":3619.46256,"input_tput_per_gpu":4805.28102,"total_input_tokens":69570139,"output_tput_per_gpu":35.47447,"total_output_tokens":513594,"avg_output_tokens_expected":2120.04183,"theoretical_cache_hit_rate":0.98034,"stdev_output_tokens_expected":2902.07726,"avg_isl":264525.24335,"avg_osl":1952.8289,"p50_itl":"0.0034500000000000004","p75_isl":386683.5,"p75_osl":2652.0,"p75_qps":0.0,"p90_isl":470981.8,"p90_itl":"0.0038799999999999998","p90_osl":5089.4,"p90_qps":0.0,"p95_isl":503534.0,"p95_osl":6909.3,"p95_qps":1.0,"std_qps":0.26781,"mean_itl":"0.0035800000000000003","mean_qps":0.07256,"p50_e2el":3.60458,"p50_tpot":"0.0034500000000000004","p50_ttft":0.37662,"p90_e2el":18.28342,"p90_tpot":"0.0038799999999999998","p90_ttft":1.63372,"mean_e2el":7.53637,"mean_tpot":0.00357,"mean_ttft":0.6724,"stdev_isl":154473.70587,"stdev_osl":2649.4789,"median_qps":0.0,"p50_intvty":289.8550724638,"p90_intvty":257.7319587629,"mean_intvty":279.3296089385},{"ep":1,"hw":"GB300 NVL72","tp":8,"conc":1,"date":"2026-10-23","num_gpus":8,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"1200388.3632075472","tput_per_gpu":2544.82333,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":269,"duration_seconds":3602.17618,"input_tput_per_gpu":2526.4358,"total_input_tokens":72805335,"output_tput_per_gpu":18.38753,"total_output_tokens":529881,"avg_output_tokens_expected":2161.39033,"theoretical_cache_hit_rate":0.98101,"stdev_output_tokens_expected":2908.29948,"avg_isl":270651.80297,"avg_osl":1969.81784,"p50_itl":0.00307,"p75_isl":390520.0,"p75_osl":2759.0,"p75_qps":0.0,"p90_isl":485900.6,"p90_itl":"0.0033399999999999997","p90_osl":5256.6,"p90_qps":0.0,"p95_isl":517698.8,"p95_osl":6907.2,"p95_qps":1.0,"std_qps":0.27104,"mean_itl":0.00314,"mean_qps":0.07457,"p50_e2el":3.59667,"p50_tpot":0.00307,"p50_ttft":0.6299199999999999,"p90_e2el":16.79596,"p90_tpot":0.00333,"p90_ttft":1.75081,"mean_e2el":"6.9716700000000005","mean_tpot":0.00313,"mean_ttft":0.84649,"stdev_isl":158035.69552,"stdev_osl":2641.71529,"median_qps":0.0,"p50_intvty":325.7328990228,"p90_intvty":299.4011976048,"mean_intvty":318.4713375796},{"ep":1,"hw":"GB300 NVL72","tp":4,"conc":4,"date":"2026-10-23","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"3459135.0849056607","tput_per_gpu":7333.36638,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":579,"duration_seconds":3598.67227,"input_tput_per_gpu":7281.19804,"total_input_tokens":104810582,"output_tput_per_gpu":52.16834,"total_output_tokens":750947,"avg_output_tokens_expected":1858.59758,"theoretical_cache_hit_rate":0.97779,"stdev_output_tokens_expected":2650.02445,"avg_isl":181020.00345,"avg_osl":1296.97237,"p50_itl":0.0037,"p75_isl":233342.5,"p75_osl":1531.0,"p75_qps":0.0,"p90_isl":528823.4,"p90_itl":0.00463,"p90_osl":3856.6,"p90_qps":1.0,"p95_isl":585564.9,"p95_osl":5417.3,"p95_qps":1.0,"std_qps":0.39889,"mean_itl":0.00404,"mean_qps":0.16109,"p50_e2el":"2.2756399999999997","p50_tpot":0.00369,"p50_ttft":0.40044,"p90_e2el":15.59134,"p90_tpot":0.00462,"p90_ttft":"1.2066700000000001","mean_e2el":5.62513,"mean_tpot":0.004030000000000001,"mean_ttft":0.65523,"stdev_isl":183335.10369,"stdev_osl":2218.6618,"median_qps":0.0,"p50_intvty":270.2702702703,"p90_intvty":215.9827213823,"mean_intvty":247.5247524752},{"ep":8,"hw":"GB300 NVL72","tp":8,"conc":880,"date":"2026-10-23","num_gpus":24,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":67818589.11792453,"tput_per_gpu":143775.40893,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":8,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39763988786/attempts/2","total_requests":94632,"num_prefill_gpu":16,"duration_seconds":3629.82694,"input_tput_per_gpu":214047.02169,"total_input_tokens":24909589846,"output_tput_per_gpu":3232.18341,"total_output_tokens":94757902,"avg_output_tokens_expected":1342.92307,"theoretical_cache_hit_rate":0.96883,"stdev_output_tokens_expected":2549.11578,"avg_isl":131364.21426,"avg_osl":991.82233,"p50_itl":0.01587,"p75_isl":163264.0,"p75_osl":998.0,"p75_qps":31.0,"p90_isl":279195.9,"p90_itl":0.01758,"p90_osl":2277.0,"p90_qps":37.0,"p95_isl":417944.95,"p95_osl":3770.45,"p95_qps":41.0,"std_qps":10.35858,"mean_itl":0.01609,"mean_qps":26.14033,"p50_e2el":"13.565190000000001","p50_tpot":0.01583,"p50_ttft":5.2005,"p90_e2el":43.1379,"p90_tpot":0.01742,"p90_ttft":11.89887,"mean_e2el":21.79525,"mean_tpot":"0.015960000000000002","mean_ttft":6.51867,"stdev_isl":132779.2821,"stdev_osl":1929.9353,"median_qps":26.0,"p50_intvty":63.0119722747,"p90_intvty":56.8828213879,"mean_intvty":62.1504039776},{"ep":8,"hw":"B300","tp":8,"conc":64,"date":"2026-10-26","num_gpus":24,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":9529227.978947368,"tput_per_gpu":18105.53316,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":10987,"num_prefill_gpu":8,"duration_seconds":3622.85929,"input_tput_per_gpu":53933.874,"total_input_tokens":3127103882,"output_tput_per_gpu":191.362755,"total_output_tokens":11119619,"avg_output_tokens_expected":1272.27891,"theoretical_cache_hit_rate":0.97268,"stdev_output_tokens_expected":2258.86985,"avg_isl":142273.47711,"avg_osl":1009.60089,"p50_itl":0.00598,"p75_isl":177477.5,"p75_osl":1084.0,"p75_qps":4.0,"p90_isl":325371.0,"p90_itl":0.006900000000000001,"p90_osl":2296.6,"p90_qps":6.0,"p95_isl":546579.1,"p95_osl":3751.7,"p95_qps":7.0,"std_qps":2.01677,"mean_itl":0.00753,"mean_qps":3.03369,"p50_e2el":4.78997,"p50_tpot":0.00597,"p50_ttft":1.60627,"p90_e2el":"16.997709999999998","p90_tpot":0.00687,"p90_ttft":3.70601,"mean_e2el":8.044310000000001,"mean_tpot":0.0075,"mean_ttft":2.13,"stdev_isl":162714.86195,"stdev_osl":1915.70341,"median_qps":3.0,"p50_intvty":167.2240802676,"p90_intvty":144.9275362319,"mean_intvty":132.802124834},{"ep":8,"hw":"B300","tp":8,"conc":240,"date":"2026-10-26","num_gpus":16,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":34786940.67894737,"tput_per_gpu":66095.18729,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"num_decode_gpu":8,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":27611,"num_prefill_gpu":8,"duration_seconds":3629.50955,"input_tput_per_gpu":131227.81006,"total_input_tokens":7627529880,"output_tput_per_gpu":962.56452,"total_output_tokens":28127868,"avg_output_tokens_expected":1337.60487,"theoretical_cache_hit_rate":0.97013,"stdev_output_tokens_expected":2372.80225,"avg_isl":138000.82264,"avg_osl":1012.24501,"p50_itl":0.01085,"p75_isl":169415.0,"p75_osl":972.0,"p75_qps":10.0,"p90_isl":289025.0,"p90_itl":0.01314,"p90_osl":2339.0,"p90_qps":13.0,"p95_isl":425784.0,"p95_osl":3880.0,"p95_qps":15.0,"std_qps":4.32537,"mean_itl":0.01286,"mean_qps":7.61235,"p50_e2el":"10.663870000000001","p50_tpot":0.01083,"p50_ttft":4.04582,"p90_e2el":37.74457,"p90_tpot":0.01304,"p90_ttft":14.56736,"mean_e2el":18.02309,"mean_tpot":0.0128,"mean_ttft":7.22555,"stdev_isl":135117.72727,"stdev_osl":2090.13992,"median_qps":7.0,"p50_intvty":92.1658986175,"p90_intvty":76.103500761,"mean_intvty":77.7604976672},{"ep":8,"hw":"B300","tp":8,"conc":64,"date":"2026-10-26","num_gpus":16,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"13184726.689473685","tput_per_gpu":25050.98071,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"num_decode_gpu":8,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":10480,"num_prefill_gpu":8,"duration_seconds":3626.72463,"input_tput_per_gpu":49740.77294,"total_input_tokens":2888023896,"output_tput_per_gpu":361.18848,"total_output_tokens":10498512,"avg_output_tokens_expected":1249.89214,"theoretical_cache_hit_rate":0.97149,"stdev_output_tokens_expected":2216.76772,"avg_isl":137706.93597,"avg_osl":999.94742,"p50_itl":0.00754,"p75_isl":170735.75,"p75_osl":1067.0,"p75_qps":4.0,"p90_isl":307686.6,"p90_itl":0.00853,"p90_osl":2260.2,"p90_qps":5.0,"p95_isl":524908.5,"p95_osl":3708.55,"p95_qps":6.0,"std_qps":1.94363,"mean_itl":0.00842,"mean_qps":2.89156,"p50_e2el":5.51779,"p50_tpot":0.00752,"p50_ttft":1.63853,"p90_e2el":19.8217,"p90_tpot":0.008490000000000001,"p90_ttft":3.66244,"mean_e2el":9.49949,"mean_tpot":0.00838,"mean_ttft":2.10397,"stdev_isl":157886.49023,"stdev_osl":1915.13695,"median_qps":3.0,"p50_intvty":132.625994695,"p90_intvty":117.2332942556,"mean_intvty":118.7648456057},{"ep":8,"hw":"B300","tp":8,"conc":384,"date":"2026-10-26","num_gpus":8,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"47227877.510526314","tput_per_gpu":89732.96727,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":20829,"duration_seconds":3627.40117,"input_tput_per_gpu":89049.60207,"total_input_tokens":2592031577,"output_tput_per_gpu":683.36521,"total_output_tokens":20072932,"avg_output_tokens_expected":1212.08671,"theoretical_cache_hit_rate":0.97092,"stdev_output_tokens_expected":2162.39164,"avg_isl":124064.95958,"avg_osl":952.0725,"p50_itl":"0.020550000000000002","p75_isl":153746.0,"p75_osl":991.0,"p75_qps":9.0,"p90_isl":276284.8,"p90_itl":0.0257,"p90_osl":2179.4,"p90_qps":13.0,"p95_isl":396037.8,"p95_osl":3580.4,"p95_qps":16.0,"std_qps":5.26299,"mean_itl":0.02097,"mean_qps":5.74379,"p50_e2el":14.84135,"p50_tpot":0.02051,"p50_ttft":3.45397,"p90_e2el":81.79131,"p90_tpot":0.0256,"p90_ttft":"29.048740000000002","mean_e2el":44.53749,"mean_tpot":0.02085,"mean_ttft":24.32345,"stdev_isl":125628.10186,"stdev_osl":1892.31103,"median_qps":5.0,"p50_intvty":48.6618004866,"p90_intvty":38.9105058366,"mean_intvty":47.6871721507},{"ep":1,"hw":"B300","tp":4,"conc":4,"date":"2026-10-26","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"3943252.3052631575","tput_per_gpu":7492.17938,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":588,"duration_seconds":3599.09181,"input_tput_per_gpu":7439.83715,"total_input_tokens":107106628,"output_tput_per_gpu":52.34223,"total_output_tokens":753538,"avg_output_tokens_expected":1878.04932,"theoretical_cache_hit_rate":0.97818,"stdev_output_tokens_expected":2655.20114,"avg_isl":182154.12925,"avg_osl":1281.52721,"p50_itl":0.00379,"p75_isl":244719.0,"p75_osl":1489.25,"p75_qps":0.0,"p90_isl":526876.8,"p90_itl":0.00496,"p90_osl":3829.2,"p90_qps":1.0,"p95_isl":583890.95,"p95_osl":5396.7,"p95_qps":1.0,"std_qps":0.39961,"mean_itl":"0.0041600000000000005","mean_qps":0.1636,"p50_e2el":2.27397,"p50_tpot":0.00377,"p50_ttft":0.45019,"p90_e2el":15.67358,"p90_tpot":0.00492,"p90_ttft":1.30179,"mean_e2el":5.7025,"mean_tpot":0.00414,"mean_ttft":0.65665,"stdev_isl":182153.66739,"stdev_osl":2205.14543,"median_qps":0.0,"p50_intvty":263.8522427441,"p90_intvty":201.6129032258,"mean_intvty":240.3846153846},{"ep":1,"hw":"B300","tp":4,"conc":8,"date":"2026-10-26","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":6633402.384210526,"tput_per_gpu":12603.46453,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":1205,"duration_seconds":3611.86532,"input_tput_per_gpu":12509.36634,"total_input_tokens":180728586,"output_tput_per_gpu":94.09819,"total_output_tokens":1359480,"avg_output_tokens_expected":1962.1192,"theoretical_cache_hit_rate":0.97695,"stdev_output_tokens_expected":2591.89515,"avg_isl":149982.22905,"avg_osl":1128.19917,"p50_itl":"0.0047599999999999995","p75_isl":171437.0,"p75_osl":1132.0,"p75_qps":1.0,"p90_isl":339730.8,"p90_itl":0.00679,"p90_osl":2794.2,"p90_qps":1.0,"p95_isl":417177.0,"p95_osl":4470.8,"p95_qps":2.0,"std_qps":0.60668,"mean_itl":0.00517,"mean_qps":0.33361,"p50_e2el":"3.1317199999999996","p50_tpot":0.00475,"p50_ttft":0.58586,"p90_e2el":14.86714,"p90_tpot":0.00679,"p90_ttft":1.20425,"mean_e2el":6.25841,"mean_tpot":0.00515,"mean_ttft":0.7695299999999999,"stdev_isl":124980.1789,"stdev_osl":1913.95935,"median_qps":0.0,"p50_intvty":210.0840336134,"p90_intvty":147.2754050074,"mean_intvty":193.4235976789},{"ep":1,"hw":"B300","tp":8,"conc":1,"date":"2026-10-26","num_gpus":8,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":1330082.3,"tput_per_gpu":2527.15637,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1792998000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39910889415/attempts/2","total_requests":268,"duration_seconds":3600.30388,"input_tput_per_gpu":2508.80375,"total_input_tokens":72259647,"output_tput_per_gpu":18.35262,"total_output_tokens":528600,"avg_output_tokens_expected":2167.50746,"theoretical_cache_hit_rate":0.98091,"stdev_output_tokens_expected":2911.99254,"avg_isl":269625.54851,"avg_osl":1972.38806,"p50_itl":0.00311,"p75_isl":390518.0,"p75_osl":2763.0,"p75_qps":0.0,"p90_isl":482996.0,"p90_itl":"0.0033900000000000002","p90_osl":5264.9,"p90_qps":0.0,"p95_isl":516148.4,"p95_osl":6907.55,"p95_qps":1.0,"std_qps":0.26756,"mean_itl":0.00321,"mean_qps":0.07433,"p50_e2el":3.51602,"p50_tpot":0.00311,"p50_ttft":0.65113,"p90_e2el":16.99419,"p90_tpot":"0.0033799999999999998","p90_ttft":"1.7785199999999999","mean_e2el":7.02149,"mean_tpot":0.00319,"mean_ttft":0.82872,"stdev_isl":157434.58967,"stdev_osl":2646.30354,"median_qps":0.0,"p50_intvty":321.5434083601,"p90_intvty":294.9852507375,"mean_intvty":311.5264797508},{"ep":1,"hw":"H200","tp":8,"conc":1,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"2146225.6131386864","tput_per_gpu":2940.32909,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":290,"duration_seconds":3603.27839,"input_tput_per_gpu":2920.58186,"total_input_tokens":84189356,"output_tput_per_gpu":19.74723,"total_output_tokens":569238,"avg_output_tokens_expected":2129.60345,"theoretical_cache_hit_rate":0.98173,"stdev_output_tokens_expected":2824.6987,"avg_isl":290308.12414,"avg_osl":1962.88966,"p50_itl":0.00266,"p75_isl":416685.5,"p75_osl":2737.0,"p75_qps":0.0,"p90_isl":526360.6,"p90_itl":0.00292,"p90_osl":5108.6,"p90_qps":0.0,"p95_isl":555316.85,"p95_osl":6779.25,"p95_qps":1.0,"std_qps":0.27882,"mean_itl":0.00271,"mean_qps":0.0803,"p50_e2el":2.67301,"p50_tpot":0.00266,"p50_ttft":0.35639,"p90_e2el":14.05609,"p90_tpot":0.00292,"p90_ttft":"0.45547000000000004","mean_e2el":5.641229999999999,"mean_tpot":0.00271,"mean_ttft":0.39183,"stdev_isl":169706.4322,"stdev_osl":2574.10711,"median_qps":0.0,"p50_intvty":375.9398496241,"p90_intvty":342.4657534247,"mean_intvty":369.0036900369},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":2342,"date":"2026-01-28","num_gpus":12,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":6658188.810011947,"tput_per_gpu":12650.5587390227,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":18971.458682817,"output_tput_per_gpu":9490.1087671255,"p50_itl":4.488675082917299,"mean_itl":4.0658557910685,"p50_e2el":26.5841890994925,"p50_tpot":0.0252666868775,"p50_ttft":"3.4066320805577996","mean_e2el":27.3197190430005,"mean_tpot":0.0248382085416,"mean_ttft":4.4401661453876,"p50_intvty":39.5778047532,"mean_intvty":40.2605525405},{"ep":1,"hw":"H200","tp":8,"conc":128,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":25660696.89051095,"tput_per_gpu":35155.15474,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":9525,"duration_seconds":3629.63366,"input_tput_per_gpu":34841.79715,"total_input_tokens":1016791801,"output_tput_per_gpu":313.35759,"total_output_tokens":9241661,"avg_output_tokens_expected":1204.25558,"theoretical_cache_hit_rate":0.9599800000000001,"stdev_output_tokens_expected":2363.78767,"avg_isl":106215.60924,"avg_osl":955.27412,"p50_itl":0.04261,"p75_isl":129041.0,"p75_osl":1012.0,"p75_qps":4.0,"p90_isl":234796.6,"p90_itl":0.06559000000000001,"p90_osl":2061.6,"p90_qps":5.0,"p95_isl":344817.8,"p95_osl":3508.2,"p95_qps":6.0,"std_qps":1.98688,"mean_itl":"0.044020000000000004","mean_qps":2.62514,"p50_e2el":"22.900389999999998","p50_tpot":0.04261,"p50_ttft":0.71174,"p90_e2el":"103.71461000000001","p90_tpot":0.06559000000000001,"p90_ttft":14.71938,"mean_e2el":46.87343,"mean_tpot":"0.044020000000000004","mean_ttft":5.02515,"stdev_isl":117055.47482,"stdev_osl":1919.50179,"median_qps":2.0,"p50_intvty":23.4686693264,"p90_intvty":15.2462265589,"mean_intvty":22.7169468423},{"ep":1,"hw":"H200","tp":8,"conc":4,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":3098466.474452555,"tput_per_gpu":4244.89907,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":658,"duration_seconds":3598.11874,"input_tput_per_gpu":4215.73678,"total_input_tokens":121349772,"output_tput_per_gpu":29.16229,"total_output_tokens":839435,"avg_output_tokens_expected":1800.73556,"theoretical_cache_hit_rate":0.97843,"stdev_output_tokens_expected":2560.22056,"avg_isl":184422.1459,"avg_osl":1275.73708,"p50_itl":"0.0027400000000000002","p75_isl":249212.5,"p75_osl":1509.5,"p75_qps":0.0,"p90_isl":518664.6,"p90_itl":0.00361,"p90_osl":3752.6,"p90_qps":1.0,"p95_isl":582395.45,"p95_osl":5367.7,"p95_qps":1.0,"std_qps":0.4165,"mean_itl":"0.0030800000000000003","mean_qps":0.18301,"p50_e2el":"1.5649600000000001","p50_tpot":"0.0027400000000000002","p50_ttft":0.2553,"p90_e2el":10.62788,"p90_tpot":0.00361,"p90_ttft":0.47524,"mean_e2el":"4.0141100000000005","mean_tpot":"0.0030800000000000003","mean_ttft":0.32503,"stdev_isl":177204.95084,"stdev_osl":2146.5226,"median_qps":0.0,"p50_intvty":364.9635036496,"p90_intvty":277.0083102493,"mean_intvty":324.6753246753},{"ep":1,"hw":"H200","tp":8,"conc":32,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"18890945.058394164","tput_per_gpu":25880.59473,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":4351,"duration_seconds":3618.60745,"input_tput_per_gpu":25725.66403,"total_input_tokens":744728636,"output_tput_per_gpu":154.9307,"total_output_tokens":4485067,"avg_output_tokens_expected":1393.41667,"theoretical_cache_hit_rate":0.97925,"stdev_output_tokens_expected":2125.25605,"avg_isl":171162.63755,"avg_osl":1030.81292,"p50_itl":0.007,"p75_isl":228525.5,"p75_osl":1102.0,"p75_qps":2.0,"p90_isl":335583.0,"p90_itl":0.01283,"p90_osl":2631.0,"p90_qps":3.0,"p95_isl":446553.5,"p95_osl":4076.5,"p95_qps":4.0,"std_qps":1.21556,"mean_itl":0.007940000000000001,"mean_qps":1.20265,"p50_e2el":3.54725,"p50_tpot":0.007,"p50_ttft":0.24267,"p90_e2el":18.87089,"p90_tpot":0.01283,"p90_ttft":0.51647,"mean_e2el":8.07175,"mean_tpot":0.007940000000000001,"mean_ttft":"0.35902999999999996","stdev_isl":150464.55754,"stdev_osl":1722.56793,"median_qps":1.0,"p50_intvty":142.8571428571,"p90_intvty":77.9423226812,"mean_intvty":125.9445843829},{"ep":1,"hw":"H200","tp":8,"conc":2,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"2283572.4963503648","tput_per_gpu":3128.49432,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":486,"duration_seconds":3599.35707,"input_tput_per_gpu":3104.34295,"total_input_tokens":89389110,"output_tput_per_gpu":24.15136,"total_output_tokens":695435,"avg_output_tokens_expected":2084.77572,"theoretical_cache_hit_rate":0.97649,"stdev_output_tokens_expected":2494.70745,"avg_isl":183928.20988,"avg_osl":1430.93621,"p50_itl":0.00271,"p75_isl":299492.25,"p75_osl":1833.0,"p75_qps":0.0,"p90_isl":391441.0,"p90_itl":0.00327,"p90_osl":4624.5,"p90_qps":1.0,"p95_isl":443015.25,"p95_osl":6012.25,"p95_qps":1.0,"std_qps":0.35741,"mean_itl":0.00294,"mean_qps":0.1348,"p50_e2el":1.51895,"p50_tpot":0.0027,"p50_ttft":0.2725,"p90_e2el":"13.290209999999998","p90_tpot":0.00327,"p90_ttft":0.45772,"mean_e2el":4.31859,"mean_tpot":0.00294,"mean_ttft":0.34287,"stdev_isl":141749.55594,"stdev_osl":2247.79332,"median_qps":0.0,"p50_intvty":369.0036900369,"p90_intvty":305.8103975535,"mean_intvty":340.1360544218},{"ep":1,"hw":"H200","tp":4,"conc":2,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":4357039.934306569,"tput_per_gpu":5969.14471,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":457,"duration_seconds":3607.5154900000002,"input_tput_per_gpu":5921.88185,"total_input_tokens":85453122,"output_tput_per_gpu":47.26286,"total_output_tokens":682006,"avg_output_tokens_expected":2130.84683,"theoretical_cache_hit_rate":0.97587,"stdev_output_tokens_expected":2596.33208,"avg_isl":186987.13786,"avg_osl":1492.35449,"p50_itl":0.00289,"p75_isl":337845.0,"p75_osl":1889.0,"p75_qps":0.0,"p90_isl":423551.8,"p90_itl":0.00373,"p90_osl":4705.2,"p90_qps":1.0,"p95_isl":482619.0,"p95_osl":6088.6,"p95_qps":1.0,"std_qps":0.35496,"mean_itl":0.00326,"mean_qps":0.12646,"p50_e2el":1.85201,"p50_tpot":0.00289,"p50_ttft":"0.29167000000000004","p90_e2el":14.37367,"p90_tpot":0.00373,"p90_ttft":0.57592,"mean_e2el":4.91558,"mean_tpot":0.00326,"mean_ttft":"0.41337999999999997","stdev_isl":154396.96108,"stdev_osl":2366.31197,"median_qps":0.0,"p50_intvty":346.0207612457,"p90_intvty":268.0965147453,"mean_intvty":306.7484662577},{"ep":1,"hw":"H200","tp":8,"conc":64,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"26906271.547445256","tput_per_gpu":36861.59202,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":8247,"duration_seconds":3628.55215,"input_tput_per_gpu":36583.1878,"total_input_tokens":1062999960,"output_tput_per_gpu":278.40423,"total_output_tokens":8105359,"avg_output_tokens_expected":1186.42645,"theoretical_cache_hit_rate":0.96931,"stdev_output_tokens_expected":2082.61137,"avg_isl":128768.28398,"avg_osl":979.94834,"p50_itl":"0.016550000000000002","p75_isl":151184.5,"p75_osl":1041.0,"p75_qps":3.0,"p90_isl":278787.4,"p90_itl":0.02189,"p90_osl":2199.2,"p90_qps":5.0,"p95_isl":487385.1,"p95_osl":3651.4,"p95_qps":5.0,"std_qps":1.64232,"mean_itl":"0.016890000000000002","mean_qps":2.27601,"p50_e2el":7.93914,"p50_tpot":"0.016550000000000002","p50_ttft":0.26853,"p90_e2el":36.64113,"p90_tpot":0.02189,"p90_ttft":0.7203999999999999,"mean_e2el":"16.750049999999998","mean_tpot":"0.016890000000000002","mean_ttft":0.46863,"stdev_isl":148329.82028,"stdev_osl":1934.24187,"median_qps":2.0,"p50_intvty":60.4229607251,"p90_intvty":45.6829602558,"mean_intvty":59.2066311427},{"ep":1,"hw":"H200","tp":8,"conc":16,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":9890234.416058395,"tput_per_gpu":13549.62115,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":2537,"duration_seconds":3600.59617,"input_tput_per_gpu":13463.86167,"total_input_tokens":388204349,"output_tput_per_gpu":85.75948,"total_output_tokens":2470283,"avg_output_tokens_expected":1583.92733,"theoretical_cache_hit_rate":0.97647,"stdev_output_tokens_expected":2462.13599,"avg_isl":152866.94127,"avg_osl":973.70201,"p50_itl":"0.0046500000000000005","p75_isl":179049.0,"p75_osl":914.0,"p75_qps":1.0,"p90_isl":278216.8,"p90_itl":"0.0077599999999999995","p90_osl":2374.8,"p90_qps":2.0,"p95_isl":524592.0,"p95_osl":4163.6,"p95_qps":2.0,"std_qps":0.91462,"mean_itl":0.00526,"mean_qps":0.70444,"p50_e2el":2.10827,"p50_tpot":"0.0046500000000000005","p50_ttft":0.26618,"p90_e2el":11.42833,"p90_tpot":0.00775,"p90_ttft":0.5290199999999999,"mean_e2el":5.05437,"mean_tpot":0.00526,"mean_ttft":"0.36149000000000003","stdev_isl":152360.00797,"stdev_osl":1934.69583,"median_qps":0.0,"p50_intvty":215.0537634409,"p90_intvty":128.8659793814,"mean_intvty":190.1140684411},{"ep":1,"hw":"H200","tp":8,"conc":8,"date":"2026-11-09","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":5404831.160583942,"tput_per_gpu":7404.61869,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":1373,"duration_seconds":3604.2318,"input_tput_per_gpu":7350.9714,"total_input_tokens":211956839,"output_tput_per_gpu":53.64728,"total_output_tokens":1546858,"avg_output_tokens_expected":1934.84373,"theoretical_cache_hit_rate":0.97811,"stdev_output_tokens_expected":2515.18682,"avg_isl":154374.97378,"avg_osl":1126.62637,"p50_itl":"0.0033799999999999998","p75_isl":171406.0,"p75_osl":1150.0,"p75_qps":1.0,"p90_isl":364299.4,"p90_itl":0.00487,"p90_osl":2917.2,"p90_qps":1.0,"p95_isl":478921.6,"p95_osl":4493.8,"p95_qps":2.0,"std_qps":0.64211,"mean_itl":0.00365,"mean_qps":0.38079,"p50_e2el":1.86612,"p50_tpot":"0.0033799999999999998","p50_ttft":0.24915,"p90_e2el":10.09778,"p90_tpot":0.00487,"p90_ttft":"0.43945999999999996","mean_e2el":4.154850000000001,"mean_tpot":0.00365,"mean_ttft":0.31443,"stdev_isl":132405.23357,"stdev_osl":1877.41616,"median_qps":0.0,"p50_intvty":295.8579881657,"p90_intvty":205.3388090349,"mean_intvty":273.9726027397},{"ep":1,"hw":"H200","tp":4,"conc":1,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"4113436.0072992705","tput_per_gpu":5635.40733,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":283,"duration_seconds":3600.3405,"input_tput_per_gpu":5596.95729,"total_input_tokens":80603808,"output_tput_per_gpu":38.45004,"total_output_tokens":553733,"avg_output_tokens_expected":2123.5053,"theoretical_cache_hit_rate":0.98253,"stdev_output_tokens_expected":2853.71305,"avg_isl":284819.10954,"avg_osl":1956.65371,"p50_itl":0.00283,"p75_isl":406478.5,"p75_osl":2715.0,"p75_qps":0.0,"p90_isl":516411.8,"p90_itl":0.0032,"p90_osl":5089.4,"p90_qps":0.0,"p95_isl":545073.7,"p95_osl":6875.5,"p95_qps":1.0,"std_qps":0.27396,"mean_itl":0.00296,"mean_qps":0.07842,"p50_e2el":"3.0391399999999997","p50_tpot":0.00283,"p50_ttft":0.35547,"p90_e2el":"14.836139999999999","p90_tpot":0.0032,"p90_ttft":0.56087,"mean_e2el":"6.0673699999999995","mean_tpot":0.00296,"mean_ttft":0.45275,"stdev_isl":166217.36705,"stdev_osl":2591.98771,"median_qps":0.0,"p50_intvty":353.3568904594,"p90_intvty":312.5,"mean_intvty":337.8378378378},{"ep":1,"hw":"H200","tp":4,"conc":4,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":6061915.445255475,"tput_per_gpu":8304.82416,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":643,"duration_seconds":3593.42783,"input_tput_per_gpu":8247.87227,"total_input_tokens":118552535,"output_tput_per_gpu":56.95189,"total_output_tokens":818610,"avg_output_tokens_expected":1808.88336,"theoretical_cache_hit_rate":0.97865,"stdev_output_tokens_expected":2579.33483,"avg_isl":184374.08243,"avg_osl":1273.11042,"p50_itl":0.00299,"p75_isl":249037.0,"p75_osl":1508.0,"p75_qps":0.0,"p90_isl":517914.4,"p90_itl":"0.0040999999999999995","p90_osl":3738.0,"p90_qps":1.0,"p95_isl":580167.8,"p95_osl":5358.3,"p95_qps":1.0,"std_qps":0.41688,"mean_itl":"0.0034300000000000003","mean_qps":0.17908,"p50_e2el":1.6684,"p50_tpot":0.00299,"p50_ttft":0.26365,"p90_e2el":12.1024,"p90_tpot":"0.0040999999999999995","p90_ttft":0.56743,"mean_e2el":4.4286,"mean_tpot":"0.0034300000000000003","mean_ttft":0.38699,"stdev_isl":176739.0453,"stdev_osl":2156.48977,"median_qps":0.0,"p50_intvty":334.4481605351,"p90_intvty":243.9024390244,"mean_intvty":291.5451895044},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":700,"date":"2026-11-06","num_gpus":22,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"106709830.32075472","tput_per_gpu":226224.84028,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":119274,"num_prefill_gpu":6,"duration_seconds":3627.87261,"input_tput_per_gpu":823825.2741033332,"total_input_tokens":35871139515,"output_tput_per_gpu":2124.67761,"total_output_tokens":123460622,"avg_output_tokens_expected":1367.14924,"theoretical_cache_hit_rate":0.97383,"stdev_output_tokens_expected":2489.18683,"avg_isl":150346.252,"avg_osl":1033.99697,"p50_itl":"0.0030600000000000002","p75_isl":196764.75,"p75_osl":1029.0,"p75_qps":38.0,"p90_isl":331829.8,"p90_itl":0.00329,"p90_osl":2429.0,"p90_qps":43.0,"p95_isl":464257.75,"p95_osl":3987.35,"p95_qps":47.0,"std_qps":9.43849,"mean_itl":"0.0033900000000000002","mean_qps":32.92106,"p50_e2el":1.74723,"p50_tpot":"0.0030499999999999998","p50_ttft":"0.28837999999999997","p90_e2el":8.122069999999999,"p90_tpot":0.00328,"p90_ttft":0.75521,"mean_e2el":3.61671,"mean_tpot":"0.0033799999999999998","mean_ttft":"0.45297000000000004","stdev_isl":145867.10643,"stdev_osl":2029.29819,"median_qps":33.0,"p50_intvty":326.7973856209,"p90_intvty":303.9513677812,"mean_intvty":294.9852507375},{"ep":1,"hw":"GB300 NVL72","tp":4,"conc":16,"date":"2026-11-06","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"13845718.183962265","tput_per_gpu":29352.92255,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":2712,"duration_seconds":3599.43527,"input_tput_per_gpu":29163.44436,"total_input_tokens":419887721,"output_tput_per_gpu":189.4782,"total_output_tokens":2728058,"avg_output_tokens_expected":1643.11821,"theoretical_cache_hit_rate":0.97626,"stdev_output_tokens_expected":2531.47259,"avg_isl":154825.85583,"avg_osl":1005.92109,"p50_itl":0.0018,"p75_isl":183462.75,"p75_osl":953.5,"p75_qps":1.0,"p90_isl":281967.7,"p90_itl":0.0024,"p90_osl":2517.2,"p90_qps":2.0,"p95_isl":527168.7,"p95_osl":4378.9,"p95_qps":2.0,"std_qps":0.89321,"mean_itl":0.00191,"mean_qps":0.75368,"p50_e2el":1.4923,"p50_tpot":"0.0017900000000000001","p50_ttft":0.4792,"p90_e2el":6.309270000000001,"p90_tpot":"0.0023799999999999997","p90_ttft":1.69913,"mean_e2el":2.72492,"mean_tpot":0.00189,"mean_ttft":0.84215,"stdev_isl":152278.73187,"stdev_osl":1954.23965,"median_qps":1.0,"p50_intvty":555.5555555556,"p90_intvty":416.6666666667,"mean_intvty":523.5602094241},{"ep":1,"hw":"GB300 NVL72","tp":4,"conc":1,"date":"2026-11-06","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":3888824.023584906,"tput_per_gpu":8244.30693,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":463,"duration_seconds":3600.06769,"input_tput_per_gpu":8192.73825,"total_input_tokens":117977649,"output_tput_per_gpu":51.56868,"total_output_tokens":742603,"avg_output_tokens_expected":2103.72786,"theoretical_cache_hit_rate":0.98228,"stdev_output_tokens_expected":2555.42954,"avg_isl":254811.33693,"avg_osl":1603.89417,"p50_itl":"0.0010400000000000001","p75_isl":416127.0,"p75_osl":2193.0,"p75_qps":0.0,"p90_isl":576565.2,"p90_itl":0.00107,"p90_osl":4748.4,"p90_qps":1.0,"p95_isl":638436.7,"p95_osl":6021.1,"p95_qps":1.0,"std_qps":0.35557,"mean_itl":0.00106,"mean_qps":0.12848,"p50_e2el":0.86393,"p50_tpot":0.00103,"p50_ttft":0.17864,"p90_e2el":"5.1691899999999995","p90_tpot":0.00106,"p90_ttft":0.2515,"mean_e2el":"1.9256099999999998","mean_tpot":"0.0010500000000000002","mean_ttft":0.27337,"stdev_isl":210847.89986,"stdev_osl":2325.95645,"median_qps":0.0,"p50_intvty":961.5384615385,"p90_intvty":934.5794392523,"mean_intvty":943.3962264151},{"ep":1,"hw":"GB300 NVL72","tp":4,"conc":4,"date":"2026-11-06","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":4892153.900943397,"tput_per_gpu":10371.36627,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":773,"duration_seconds":3599.57811,"input_tput_per_gpu":10300.90961,"total_input_tokens":148315715,"output_tput_per_gpu":70.45666,"total_output_tokens":1014457,"avg_output_tokens_expected":1765.15524,"theoretical_cache_hit_rate":0.97882,"stdev_output_tokens_expected":2565.85818,"avg_isl":191870.2652,"avg_osl":1312.36352,"p50_itl":0.0015,"p75_isl":274452.0,"p75_osl":1545.0,"p75_qps":0.0,"p90_isl":516412.8,"p90_itl":0.00188,"p90_osl":3758.6,"p90_qps":1.0,"p95_isl":595801.6,"p95_osl":5354.8,"p95_qps":1.0,"std_qps":0.45565,"mean_itl":0.00165,"mean_qps":0.2148,"p50_e2el":0.95438,"p50_tpot":0.0015,"p50_ttft":0.15888,"p90_e2el":6.17774,"p90_tpot":0.00184,"p90_ttft":"0.24633000000000002","mean_e2el":2.24993,"mean_tpot":0.00164,"mean_ttft":0.19329,"stdev_isl":176880.75715,"stdev_osl":2190.47135,"median_qps":0.0,"p50_intvty":666.6666666667,"p90_intvty":531.914893617,"mean_intvty":606.0606060606},{"ep":1,"hw":"H200","tp":4,"conc":128,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":7677006.226277374,"tput_per_gpu":10517.49853,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":1676,"duration_seconds":3622.48546,"input_tput_per_gpu":10397.33891,"total_input_tokens":152081130,"output_tput_per_gpu":120.15963,"total_output_tokens":1749950,"avg_output_tokens_expected":1023.77506,"theoretical_cache_hit_rate":0.94909,"stdev_output_tokens_expected":2009.23486,"avg_isl":89890.7136,"avg_osl":1038.84606,"p50_itl":0.03505,"p75_isl":105050.75,"p75_osl":1010.5,"p75_qps":1.0,"p90_isl":204903.5,"p90_itl":"0.039119999999999995","p90_osl":1871.5,"p90_qps":1.0,"p95_isl":325696.25,"p95_osl":3370.25,"p95_qps":2.0,"std_qps":0.9341,"mean_itl":0.03454,"mean_qps":0.46218,"p50_e2el":671.2901,"p50_tpot":0.03505,"p50_ttft":653.82729,"p90_e2el":767.14624,"p90_tpot":"0.039119999999999995","p90_ttft":731.71528,"mean_e2el":419.47228,"mean_tpot":0.03454,"mean_ttft":382.02494,"stdev_isl":118299.92447,"stdev_osl":2921.29359,"median_qps":0.0,"p50_intvty":28.5306704708,"p90_intvty":25.5623721881,"mean_intvty":28.95193978},{"ep":1,"hw":"H200","tp":4,"conc":16,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"19316204.386861317","tput_per_gpu":26463.20001,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":2506,"duration_seconds":3611.86681,"input_tput_per_gpu":26293.81918,"total_input_tokens":379879091,"output_tput_per_gpu":169.38083,"total_output_tokens":2447124,"avg_output_tokens_expected":1585.01506,"theoretical_cache_hit_rate":0.97612,"stdev_output_tokens_expected":2466.18018,"avg_isl":151587.82562,"avg_osl":976.50599,"p50_itl":0.0053,"p75_isl":176634.0,"p75_osl":914.75,"p75_qps":1.0,"p90_isl":275582.0,"p90_itl":0.00875,"p90_osl":2385.0,"p90_qps":2.0,"p95_isl":525650.75,"p95_osl":4200.5,"p95_qps":2.0,"std_qps":0.9022,"mean_itl":0.00604,"mean_qps":0.69371,"p50_e2el":2.46942,"p50_tpot":0.0053,"p50_ttft":"0.27485000000000004","p90_e2el":13.39731,"p90_tpot":0.00874,"p90_ttft":0.72634,"mean_e2el":5.92101,"mean_tpot":0.00604,"mean_ttft":0.49374,"stdev_isl":152659.47266,"stdev_osl":1944.71686,"median_qps":0.0,"p50_intvty":188.679245283,"p90_intvty":114.2857142857,"mean_intvty":165.5629139073},{"ep":1,"hw":"H200","tp":4,"conc":32,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"36082011.343065694","tput_per_gpu":49432.35554,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":4237,"duration_seconds":3618.3444,"input_tput_per_gpu":49130.84976,"total_input_tokens":711364877,"output_tput_per_gpu":301.50578,"total_output_tokens":4372206,"avg_output_tokens_expected":1402.9786,"theoretical_cache_hit_rate":0.97883,"stdev_output_tokens_expected":2137.7842,"avg_isl":167828.49658,"avg_osl":1029.92849,"p50_itl":0.00763,"p75_isl":223772.0,"p75_osl":1105.0,"p75_qps":2.0,"p90_isl":330589.4,"p90_itl":0.0143,"p90_osl":2642.0,"p90_qps":3.0,"p95_isl":417423.8,"p95_osl":4097.0,"p95_qps":3.0,"std_qps":1.18826,"mean_itl":0.00885,"mean_qps":1.17114,"p50_e2el":"3.8892800000000003","p50_tpot":0.00763,"p50_ttft":0.2534,"p90_e2el":21.81851,"p90_tpot":0.0143,"p90_ttft":0.6623600000000001,"mean_e2el":9.05786,"mean_tpot":0.00885,"mean_ttft":0.44653,"stdev_isl":147161.25971,"stdev_osl":1718.76295,"median_qps":1.0,"p50_intvty":131.0615989515,"p90_intvty":69.9300699301,"mean_intvty":112.9943502825},{"ep":1,"hw":"H200","tp":4,"conc":64,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":47651891.24817518,"tput_per_gpu":65283.09101,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":7408,"duration_seconds":3628.84956,"input_tput_per_gpu":64779.99534,"total_input_tokens":944977773,"output_tput_per_gpu":503.09567,"total_output_tokens":7400845,"avg_output_tokens_expected":1134.80011,"theoretical_cache_hit_rate":0.96899,"stdev_output_tokens_expected":1905.67054,"avg_isl":126931.34854,"avg_osl":985.77673,"p50_itl":"0.018920000000000003","p75_isl":148131.0,"p75_osl":1041.0,"p75_qps":3.0,"p90_isl":275111.6,"p90_itl":0.0257,"p90_osl":2258.0,"p90_qps":4.0,"p95_isl":484391.2,"p95_osl":3670.0,"p95_qps":5.0,"std_qps":1.57549,"mean_itl":0.01946,"mean_qps":2.04275,"p50_e2el":9.49706,"p50_tpot":"0.018920000000000003","p50_ttft":"0.31642000000000003","p90_e2el":"45.541650000000004","p90_tpot":0.0257,"p90_ttft":1.97492,"mean_e2el":20.04507,"mean_tpot":0.01946,"mean_ttft":1.11623,"stdev_isl":146134.27231,"stdev_osl":1959.56425,"median_qps":2.0,"p50_intvty":52.8541226216,"p90_intvty":38.9105058366,"mean_intvty":51.3874614594},{"ep":1,"hw":"H200","tp":4,"conc":8,"date":"2026-11-09","num_gpus":4,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"10591195.343065694","tput_per_gpu":14509.93762,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1794211200,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42341901521/attempts/1","total_requests":1341,"duration_seconds":3599.21194,"input_tput_per_gpu":14405.24206,"total_input_tokens":207390077,"output_tput_per_gpu":104.69556,"total_output_tokens":1507286,"avg_output_tokens_expected":1938.06154,"theoretical_cache_hit_rate":0.97822,"stdev_output_tokens_expected":2525.02562,"avg_isl":154653.30127,"avg_osl":1124.00149,"p50_itl":"0.0038399999999999997","p75_isl":173612.0,"p75_osl":1155.0,"p75_qps":1.0,"p90_isl":364829.0,"p90_itl":0.00549,"p90_osl":2902.0,"p90_qps":1.0,"p95_isl":477274.0,"p95_osl":4482.0,"p95_qps":2.0,"std_qps":0.63547,"mean_itl":0.00414,"mean_qps":0.37243,"p50_e2el":2.18654,"p50_tpot":"0.0038399999999999997","p50_ttft":0.25933,"p90_e2el":"11.337069999999999","p90_tpot":0.00549,"p90_ttft":0.5512699999999999,"mean_e2el":4.68717,"mean_tpot":0.00414,"mean_ttft":0.38894,"stdev_isl":132554.69468,"stdev_osl":1867.98697,"median_qps":0.0,"p50_intvty":260.4166666667,"p90_intvty":182.1493624772,"mean_intvty":241.5458937198},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":48167.87524496351,"tput_per_gpu":65.9899890856,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":32.8782012821,"output_tput_per_gpu":33.1117878034,"p50_itl":0.0073854443617,"p90_itl":0.0077260948718,"mean_itl":0.0074446282661,"p50_e2el":6.844503028318299,"p50_tpot":0.0074377569385,"p50_ttft":0.1002820008434,"p90_e2el":7.4845350246876,"p90_tpot":0.0075046042911,"p90_ttft":0.114125463739,"mean_e2el":6.9562054187059,"mean_tpot":0.0074448825433,"mean_ttft":"0.10416990946980001","p50_intvty":134.4491367847,"p90_intvty":133.2515294892,"mean_intvty":134.320453571},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":7480,"date":"2026-11-06","num_gpus":34,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":525179520.8679246,"tput_per_gpu":1113380.58424,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":979434,"num_prefill_gpu":18,"duration_seconds":3629.94486,"input_tput_per_gpu":2087763.813322222,"total_input_tokens":272955750606,"output_tput_per_gpu":17199.4515225,"total_output_tokens":1003291240,"avg_output_tokens_expected":1359.8324,"theoretical_cache_hit_rate":0.97075,"stdev_output_tokens_expected":2531.98181,"avg_isl":139276.78165,"avg_osl":1019.90432,"p50_itl":0.00919,"p75_isl":176466.0,"p75_osl":1020.0,"p75_qps":301.0,"p90_isl":298241.0,"p90_itl":0.01013,"p90_osl":2368.0,"p90_qps":323.0,"p95_isl":428970.35,"p95_osl":3903.05,"p95_qps":337.0,"std_qps":55.17865,"mean_itl":0.009710000000000002,"mean_qps":270.18372,"p50_e2el":7.46566,"p50_tpot":0.009179999999999999,"p50_ttft":2.28285,"p90_e2el":26.50657,"p90_tpot":0.01009,"p90_ttft":7.563680000000001,"mean_e2el":12.69317,"mean_tpot":0.00968,"mean_ttft":3.55429,"stdev_isl":137876.29026,"stdev_osl":1985.30979,"median_qps":276.0,"p50_intvty":108.8139281828,"p90_intvty":98.7166831194,"mean_intvty":102.9866117405},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":1228,"date":"2026-11-06","num_gpus":22,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"180956484.32547173","tput_per_gpu":383627.74677,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":205165,"num_prefill_gpu":6,"duration_seconds":3628.65502,"input_tput_per_gpu":"1396913.9852433333","total_input_tokens":60829401308,"output_tput_per_gpu":3645.4073425,"total_output_tokens":211975235,"avg_output_tokens_expected":1366.12328,"theoretical_cache_hit_rate":0.97335,"stdev_output_tokens_expected":2522.0964,"avg_isl":148239.28887,"avg_osl":1031.59316,"p50_itl":0.00328,"p75_isl":193821.0,"p75_osl":1027.0,"p75_qps":64.0,"p90_isl":326575.0,"p90_itl":"0.0035800000000000003","p90_osl":2421.0,"p90_qps":72.0,"p95_isl":460232.8,"p95_osl":3984.0,"p95_qps":79.0,"std_qps":15.03436,"mean_itl":0.00375,"mean_qps":56.59697,"p50_e2el":"2.1533200000000003","p50_tpot":0.00327,"p50_ttft":0.44617,"p90_e2el":9.1349,"p90_tpot":"0.0035600000000000002","p90_ttft":"1.3443900000000002","mean_e2el":4.15688,"mean_tpot":"0.0037400000000000003","mean_ttft":0.7680499999999999,"stdev_isl":143517.06091,"stdev_osl":2001.46347,"median_qps":56.0,"p50_intvty":304.8780487805,"p90_intvty":279.3296089385,"mean_intvty":266.6666666667},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":2884,"date":"2026-11-06","num_gpus":26,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":337557915.1509434,"tput_per_gpu":715622.78012,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":458076,"num_prefill_gpu":10,"duration_seconds":3629.70248,"input_tput_per_gpu":1847565.1688,"total_input_tokens":134133667522,"output_tput_per_gpu":8158.787195,"total_output_tokens":474774189,"avg_output_tokens_expected":1371.32521,"theoretical_cache_hit_rate":0.97275,"stdev_output_tokens_expected":2534.842,"avg_isl":146397.36347,"avg_osl":1034.37753,"p50_itl":0.004059999999999999,"p75_isl":190166.0,"p75_osl":1027.0,"p75_qps":142.0,"p90_isl":318758.5,"p90_itl":0.00433,"p90_osl":2419.0,"p90_qps":155.0,"p95_isl":446177.0,"p95_osl":3990.0,"p95_qps":163.0,"std_qps":26.24508,"mean_itl":0.00454,"mean_qps":126.39983,"p50_e2el":3.35109,"p50_tpot":0.004059999999999999,"p50_ttft":0.9484600000000001,"p90_e2el":12.7365,"p90_tpot":0.00431,"p90_ttft":"3.7394600000000002","mean_e2el":5.89909,"mean_tpot":0.00453,"mean_ttft":1.72836,"stdev_isl":141811.11625,"stdev_osl":2014.4135,"median_qps":129.0,"p50_intvty":246.3054187192,"p90_intvty":230.9468822171,"mean_intvty":220.2643171806},{"ep":16,"hw":"GB300 NVL72","tp":16,"conc":5540,"date":"2026-11-06","num_gpus":30,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":478472891.2783019,"tput_per_gpu":1014362.52951,"model_display":"DeepSeek-V4.1-Flash","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"num_decode_gpu":16,"serving_config":"disagg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41581664540/attempts/3","total_requests":774273,"num_prefill_gpu":14,"duration_seconds":3629.60737,"input_tput_per_gpu":2157911.228442857,"total_input_tokens":219362308876,"output_tput_per_gpu":13757.41794375,"total_output_tokens":801262033,"avg_output_tokens_expected":1371.82537,"theoretical_cache_hit_rate":0.97139,"stdev_output_tokens_expected":2543.69333,"avg_isl":141620.8327,"avg_osl":1031.86397,"p50_itl":0.00596,"p75_isl":181573.0,"p75_osl":1027.0,"p75_qps":238.0,"p90_isl":304485.0,"p90_itl":0.0064,"p90_osl":2408.0,"p90_qps":255.0,"p95_isl":431162.0,"p95_osl":3964.0,"p95_qps":265.0,"std_qps":41.10782,"mean_itl":0.00666,"mean_qps":213.65011,"p50_e2el":6.84624,"p50_tpot":0.00595,"p50_ttft":3.06939,"p90_e2el":20.58448,"p90_tpot":0.00636,"p90_ttft":8.970040000000001,"mean_e2el":10.33286,"mean_tpot":0.00664,"mean_ttft":"4.2838199999999995","stdev_isl":138544.51845,"stdev_osl":2009.17051,"median_qps":219.0,"p50_intvty":167.7852348993,"p90_intvty":156.25,"mean_intvty":150.1501501502},{"ep":1,"hw":"Vera Rubin NVL72","tp":8,"conc":16,"date":"2026-10-23","num_gpus":10,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":3387653.666666667,"tput_per_gpu":11179.2571,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":8,"serving_config":"disagg","total_requests":2571,"num_prefill_gpu":2,"duration_seconds":3604.71922,"input_tput_per_gpu":55546.92835,"total_input_tokens":400843902,"output_tput_per_gpu":87.3392875,"total_output_tokens":2518671,"avg_output_tokens_expected":1577.4259,"theoretical_cache_hit_rate":0.97705,"stdev_output_tokens_expected":2451.25406,"avg_isl":155761.24465,"avg_osl":979.64566,"p50_itl":0.00088,"p75_isl":182841.5,"p75_osl":936.0,"p75_qps":1.0,"p90_isl":279886.0,"p90_itl":"0.0010500000000000002","p90_osl":2401.0,"p90_qps":2.0,"p95_isl":537686.0,"p95_osl":4217.5,"p95_qps":3.0,"std_qps":0.96546,"mean_itl":0.00091,"mean_qps":0.7131,"p50_e2el":3.24153,"p50_tpot":0.00087,"p50_ttft":"2.3193099999999998","p90_e2el":13.57597,"p90_tpot":"0.0010400000000000001","p90_ttft":12.42898,"mean_e2el":5.53188,"mean_tpot":0.0009,"mean_ttft":4.63855,"stdev_isl":154096.71935,"stdev_osl":1932.76933,"median_qps":0.0,"p50_intvty":1136.3636363636,"p90_intvty":952.380952381,"mean_intvty":1098.9010989011},{"ep":1,"hw":"Vera Rubin NVL72","tp":2,"conc":52,"date":"2026-10-23","num_gpus":10,"scenario":"agentic","decode_ep":1,"decode_tp":2,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"12038296.803030303","tput_per_gpu":39726.37945,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":8,"serving_config":"disagg","total_requests":9254,"num_prefill_gpu":2,"duration_seconds":3611.5645,"input_tput_per_gpu":197291.02795,"total_input_tokens":1425098669,"output_tput_per_gpu":335.217325,"total_output_tokens":9685274,"avg_output_tokens_expected":1342.99709,"theoretical_cache_hit_rate":0.97598,"stdev_output_tokens_expected":2293.72593,"avg_isl":153993.79155,"avg_osl":1046.60385,"p50_itl":0.00186,"p75_isl":206979.75,"p75_osl":1086.0,"p75_qps":4.0,"p90_isl":350745.6,"p90_itl":"0.0020499999999999997","p90_osl":2514.7,"p90_qps":5.0,"p95_isl":564671.5,"p95_osl":4046.4,"p95_qps":6.0,"std_qps":1.76214,"mean_itl":0.00181,"mean_qps":2.56245,"p50_e2el":2.22969,"p50_tpot":0.00186,"p50_ttft":0.55432,"p90_e2el":"10.076139999999999","p90_tpot":0.00204,"p90_ttft":6.35173,"mean_e2el":4.14827,"mean_tpot":0.0018,"mean_ttft":2.21975,"stdev_isl":168659.30619,"stdev_osl":2024.88902,"median_qps":2.0,"p50_intvty":537.6344086022,"p90_intvty":487.8048780488,"mean_intvty":552.4861878453},{"ep":1,"hw":"Vera Rubin NVL72","tp":4,"conc":24,"date":"2026-10-23","num_gpus":10,"scenario":"agentic","decode_ep":1,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":5652926.327272727,"tput_per_gpu":18654.65688,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"num_decode_gpu":8,"serving_config":"disagg","total_requests":3941,"num_prefill_gpu":2,"duration_seconds":3605.51466,"input_tput_per_gpu":92720.51335,"total_input_tokens":668992082,"output_tput_per_gpu":138.19275,"total_output_tokens":3986050,"avg_output_tokens_expected":1492.56477,"theoretical_cache_hit_rate":0.97878,"stdev_output_tokens_expected":2236.32415,"avg_isl":169654.99645,"avg_osl":1011.4306,"p50_itl":0.00103,"p75_isl":227879.0,"p75_osl":1053.0,"p75_qps":2.0,"p90_isl":338802.0,"p90_itl":0.00115,"p90_osl":2608.0,"p90_qps":3.0,"p95_isl":443042.0,"p95_osl":4223.0,"p95_qps":3.0,"std_qps":1.20008,"mean_itl":"0.0010500000000000002","mean_qps":1.09293,"p50_e2el":2.23542,"p50_tpot":0.00102,"p50_ttft":1.12987,"p90_e2el":9.35108,"p90_tpot":0.00114,"p90_ttft":7.68196,"mean_e2el":4.02781,"mean_tpot":"0.0010400000000000001","mean_ttft":2.95546,"stdev_isl":146267.57974,"stdev_osl":1752.97059,"median_qps":1.0,"p50_intvty":970.8737864078,"p90_intvty":869.5652173913,"mean_intvty":952.380952381},{"ep":1,"hw":"Vera Rubin NVL72","tp":2,"conc":56,"date":"2026-10-23","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":53111934.38787878,"tput_per_gpu":175269.38348,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"serving_config":"agg","total_requests":8708,"duration_seconds":3628.99717,"input_tput_per_gpu":174026.74445,"total_input_tokens":1263890011,"output_tput_per_gpu":1242.63902,"total_output_tokens":9034244,"avg_output_tokens_expected":1267.13949,"theoretical_cache_hit_rate":0.97408,"stdev_output_tokens_expected":2158.96146,"avg_isl":145048.82017,"avg_osl":1035.72198,"p50_itl":0.006849999999999999,"p75_isl":182401.25,"p75_osl":1059.0,"p75_qps":3.0,"p90_isl":337268.4,"p90_itl":0.00874,"p90_osl":2431.9,"p90_qps":5.0,"p95_isl":555007.5,"p95_osl":4042.3,"p95_qps":5.0,"std_qps":1.6308,"mean_itl":"0.0069299999999999995","mean_qps":2.39994,"p50_e2el":"4.4021300000000005","p50_tpot":0.00684,"p50_ttft":0.34676,"p90_e2el":19.21943,"p90_tpot":0.00873,"p90_ttft":3.76606,"mean_e2el":8.5822,"mean_tpot":0.00694,"mean_ttft":1.36592,"stdev_isl":162512.88443,"stdev_osl":2031.67499,"median_qps":2.0,"p50_intvty":145.9854014599,"p90_intvty":114.4164759725,"mean_intvty":144.3001443001},{"ep":1,"hw":"Vera Rubin NVL72","tp":2,"conc":20,"date":"2026-10-23","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"22592645.675757576","tput_per_gpu":74555.73073,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"serving_config":"agg","total_requests":3188,"duration_seconds":3602.10171,"input_tput_per_gpu":74090.02664,"total_input_tokens":534195968,"output_tput_per_gpu":465.70409,"total_output_tokens":3355030,"avg_output_tokens_expected":1578.49952,"theoretical_cache_hit_rate":0.97884,"stdev_output_tokens_expected":2332.8884,"avg_isl":167427.7362,"avg_osl":1052.39241,"p50_itl":0.00215,"p75_isl":215292.0,"p75_osl":1097.25,"p75_qps":1.0,"p90_isl":331004.3,"p90_itl":0.00257,"p90_osl":2681.9,"p90_qps":2.0,"p95_isl":441276.9,"p95_osl":4379.3,"p95_qps":3.0,"std_qps":1.00601,"mean_itl":0.00221,"mean_qps":0.88503,"p50_e2el":2.23442,"p50_tpot":0.00214,"p50_ttft":0.28901,"p90_e2el":"10.346540000000001","p90_tpot":0.00256,"p90_ttft":5.9028,"mean_e2el":4.118189999999999,"mean_tpot":0.0022,"mean_ttft":"1.8235899999999998","stdev_isl":149119.7886,"stdev_osl":1833.61786,"median_qps":1.0,"p50_intvty":465.1162790698,"p90_intvty":389.1050583658,"mean_intvty":452.4886877828},{"ep":1,"hw":"Vera Rubin NVL72","tp":2,"conc":46,"date":"2026-10-23","num_gpus":2,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":46316727.61515152,"tput_per_gpu":152845.20113,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1792738800,"serving_config":"agg","total_requests":6756,"duration_seconds":3620.80123,"input_tput_per_gpu":151868.6895,"total_input_tokens":1100469766,"output_tput_per_gpu":976.51163,"total_output_tokens":7085125,"avg_output_tokens_expected":1277.78788,"theoretical_cache_hit_rate":0.9766,"stdev_output_tokens_expected":2047.80171,"avg_isl":162784.58777,"avg_osl":1046.70056,"p50_itl":0.00487,"p75_isl":205277.25,"p75_osl":1065.0,"p75_qps":3.0,"p90_isl":361536.0,"p90_itl":0.0058,"p90_osl":2520.0,"p90_qps":4.0,"p95_isl":622393.25,"p95_osl":4061.25,"p95_qps":4.0,"std_qps":1.36984,"mean_itl":0.00475,"mean_qps":1.86575,"p50_e2el":4.28107,"p50_tpot":0.004860000000000001,"p50_ttft":0.8889900000000001,"p90_e2el":15.62421,"p90_tpot":0.0058,"p90_ttft":5.28659,"mean_e2el":"7.2605699999999995","mean_tpot":0.00475,"mean_ttft":2.13227,"stdev_isl":174805.99786,"stdev_osl":2109.68227,"median_qps":2.0,"p50_intvty":205.3388090349,"p90_intvty":172.4137931034,"mean_intvty":210.5263157895},{"ep":8,"hw":"Vera Rubin NVL72","tp":8,"conc":56,"date":"2026-11-03","num_gpus":36,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":3929526.384848485,"tput_per_gpu":12967.43707,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"num_decode_gpu":32,"serving_config":"disagg","total_requests":10526,"num_prefill_gpu":4,"duration_seconds":3608.78172,"input_tput_per_gpu":115939.24326,"total_input_tokens":1673597686,"output_tput_per_gpu":95.96129625,"total_output_tokens":11081708,"avg_output_tokens_expected":1363.12619,"theoretical_cache_hit_rate":0.97714,"stdev_output_tokens_expected":2353.54565,"avg_isl":158996.55007,"avg_osl":1052.79384,"p50_itl":0.0009599999999999999,"p75_isl":214297.75,"p75_osl":1082.75,"p75_qps":4.0,"p90_isl":387953.0,"p90_itl":0.00103,"p90_osl":2520.0,"p90_qps":6.0,"p95_isl":574374.5,"p95_osl":4072.5,"p95_qps":6.0,"std_qps":1.94037,"mean_itl":0.001,"mean_qps":2.91713,"p50_e2el":1.7572,"p50_tpot":0.0009599999999999999,"p50_ttft":0.8966799999999999,"p90_e2el":7.13983,"p90_tpot":0.00102,"p90_ttft":5.280270000000001,"mean_e2el":3.13494,"mean_tpot":0.00099,"mean_ttft":2.11998,"stdev_isl":173152.06064,"stdev_osl":2032.00361,"median_qps":3.0,"p50_intvty":1041.6666666667,"p90_intvty":970.8737864078,"mean_intvty":1000.0},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":84,"date":"2026-11-03","num_gpus":32,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5964857.687878788,"tput_per_gpu":19684.03037,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"num_decode_gpu":28,"serving_config":"disagg","total_requests":15704,"num_prefill_gpu":4,"duration_seconds":3603.4208,"input_tput_per_gpu":156318.24216,"total_input_tokens":2253121619,"output_tput_per_gpu":164.8572571429,"total_output_tokens":16633402,"avg_output_tokens_expected":1364.91626,"theoretical_cache_hit_rate":0.97279,"stdev_output_tokens_expected":2555.92862,"avg_isl":143474.37717,"avg_osl":1059.1825,"p50_itl":0.00126,"p75_isl":190571.75,"p75_osl":1094.0,"p75_qps":6.0,"p90_isl":329136.8,"p90_itl":0.00133,"p90_osl":2419.4,"p90_qps":7.0,"p95_isl":518643.6,"p95_osl":3998.55,"p95_qps":8.0,"std_qps":2.27636,"mean_itl":"0.0012900000000000001","mean_qps":4.35803,"p50_e2el":3.00028,"p50_tpot":0.00125,"p50_ttft":1.76151,"p90_e2el":"10.860389999999999","p90_tpot":0.00132,"p90_ttft":8.49246,"mean_e2el":4.804,"mean_tpot":0.00128,"mean_ttft":3.4677,"stdev_isl":160841.54838,"stdev_osl":2009.64976,"median_qps":4.0,"p50_intvty":793.6507936508,"p90_intvty":751.8796992481,"mean_intvty":775.1937984496},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":100,"date":"2026-11-03","num_gpus":20,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"10498624.645454545","tput_per_gpu":34645.46133,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"num_decode_gpu":16,"serving_config":"disagg","total_requests":17194,"num_prefill_gpu":4,"duration_seconds":3603.34781,"input_tput_per_gpu":171964.5499,"total_input_tokens":2478592340,"output_tput_per_gpu":315.6891875,"total_output_tokens":18200607,"avg_output_tokens_expected":1259.13038,"theoretical_cache_hit_rate":0.97275,"stdev_output_tokens_expected":2332.25221,"avg_isl":144154.49226,"avg_osl":1058.54409,"p50_itl":"0.0020299999999999997","p75_isl":192987.0,"p75_osl":1099.0,"p75_qps":6.0,"p90_isl":316988.1,"p90_itl":0.00216,"p90_osl":2408.0,"p90_qps":8.0,"p95_isl":485335.9,"p95_osl":3999.0,"p95_qps":9.0,"std_qps":2.44487,"mean_itl":0.002,"mean_qps":4.77186,"p50_e2el":3.4006,"p50_tpot":"0.0020299999999999997","p50_ttft":1.66227,"p90_e2el":11.09568,"p90_tpot":0.00215,"p90_ttft":7.73234,"mean_e2el":5.15325,"mean_tpot":0.00199,"mean_ttft":3.08948,"stdev_isl":154705.98729,"stdev_osl":1986.80987,"median_qps":5.0,"p50_intvty":492.6108374384,"p90_intvty":462.962962963,"mean_intvty":500.0},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":100,"date":"2026-11-03","num_gpus":24,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":8654943.084848486,"tput_per_gpu":28561.31218,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"num_decode_gpu":20,"serving_config":"disagg","total_requests":17030,"num_prefill_gpu":4,"duration_seconds":3603.48999,"input_tput_per_gpu":170114.80878,"total_input_tokens":2452028044,"output_tput_per_gpu":250.61286,"total_output_tokens":18061619,"avg_output_tokens_expected":1261.7917,"theoretical_cache_hit_rate":0.97265,"stdev_output_tokens_expected":2339.52732,"avg_isl":143982.85637,"avg_osl":1060.57657,"p50_itl":0.00175,"p75_isl":192490.25,"p75_osl":1099.75,"p75_qps":6.0,"p90_isl":316481.5,"p90_itl":0.00185,"p90_osl":2415.0,"p90_qps":8.0,"p95_isl":482603.95,"p95_osl":4016.75,"p95_qps":9.0,"std_qps":2.45471,"mean_itl":0.00173,"mean_qps":4.72634,"p50_e2el":"4.0736799999999995","p50_tpot":0.00175,"p50_ttft":2.32508,"p90_e2el":12.19957,"p90_tpot":0.00185,"p90_ttft":9.3353,"mean_e2el":5.72693,"mean_tpot":0.00172,"mean_ttft":3.93416,"stdev_isl":154466.43335,"stdev_osl":1993.97553,"median_qps":5.0,"p50_intvty":571.4285714286,"p90_intvty":540.5405405405,"mean_intvty":578.0346820809},{"ep":1,"hw":"GB300 NVL72","tp":1,"conc":8,"date":"2026-11-06","num_gpus":1,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"18570378.108490564","tput_per_gpu":39369.20159,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42124076518/attempts/5","total_requests":1452,"duration_seconds":3609.9092,"input_tput_per_gpu":38988.1884,"total_input_tokens":140743820,"output_tput_per_gpu":381.01318,"total_output_tokens":1375423,"avg_output_tokens_expected":1753.78887,"theoretical_cache_hit_rate":0.95932,"stdev_output_tokens_expected":2805.25119,"avg_isl":96931.00551,"avg_osl":947.26102,"p50_itl":0.00342,"p75_isl":122676.5,"p75_osl":813.75,"p75_qps":1.0,"p90_isl":168414.9,"p90_itl":0.00378,"p90_osl":2161.3,"p90_qps":1.0,"p95_isl":195585.35,"p95_osl":4116.2,"p95_qps":2.0,"std_qps":0.63491,"mean_itl":"0.0035800000000000003","mean_qps":0.40261,"p50_e2el":2.62361,"p50_tpot":0.00342,"p50_ttft":0.6645,"p90_e2el":"14.309629999999999","p90_tpot":0.00378,"p90_ttft":5.65986,"mean_e2el":5.75818,"mean_tpot":"0.0035800000000000003","mean_ttft":2.46066,"stdev_isl":50074.77154,"stdev_osl":2018.06612,"median_qps":0.0,"p50_intvty":292.3976608187,"p90_intvty":264.5502645503,"mean_intvty":279.3296089385},{"ep":1,"hw":"GB300 NVL72","tp":1,"conc":20,"date":"2026-11-06","num_gpus":1,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":48367113.0754717,"tput_per_gpu":102538.27972,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42124076518/attempts/5","total_requests":3388,"duration_seconds":3606.84051,"input_tput_per_gpu":101673.28556,"total_input_tokens":366719325,"output_tput_per_gpu":864.99417,"total_output_tokens":3119896,"avg_output_tokens_expected":1445.19138,"theoretical_cache_hit_rate":0.96425,"stdev_output_tokens_expected":2262.83039,"avg_isl":108240.65083,"avg_osl":920.86659,"p50_itl":0.00497,"p75_isl":150492.0,"p75_osl":914.5,"p75_qps":2.0,"p90_isl":201410.5,"p90_itl":"0.0059900000000000005","p90_osl":2318.1,"p90_qps":2.0,"p95_isl":225901.8,"p95_osl":3834.6,"p95_qps":3.0,"std_qps":0.95283,"mean_itl":0.0054,"mean_qps":0.94083,"p50_e2el":3.8487,"p50_tpot":0.00496,"p50_ttft":1.05511,"p90_e2el":18.34506,"p90_tpot":"0.0059900000000000005","p90_ttft":"6.3117399999999995","mean_e2el":8.01518,"mean_tpot":0.0054,"mean_ttft":"3.2327199999999996","stdev_isl":61644.56761,"stdev_osl":1740.56314,"median_qps":1.0,"p50_intvty":201.2072434608,"p90_intvty":166.9449081803,"mean_intvty":185.1851851852},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":48,"date":"2026-11-03","num_gpus":4,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":25762695.53939394,"tput_per_gpu":85016.89528,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","total_requests":7674,"duration_seconds":3623.87613,"input_tput_per_gpu":84462.04643,"total_input_tokens":1224319977,"output_tput_per_gpu":554.84885,"total_output_tokens":8042814,"avg_output_tokens_expected":1286.24337,"theoretical_cache_hit_rate":0.97636,"stdev_output_tokens_expected":2145.30573,"avg_isl":159541.30532,"avg_osl":1048.0602,"p50_itl":"0.0044599999999999996","p75_isl":205624.75,"p75_osl":1067.0,"p75_qps":3.0,"p90_isl":354042.2,"p90_itl":0.00518,"p90_osl":2494.4,"p90_qps":4.0,"p95_isl":605783.85,"p95_osl":4041.7,"p95_qps":5.0,"std_qps":1.55626,"mean_itl":0.00443,"mean_qps":2.11786,"p50_e2el":2.64396,"p50_tpot":"0.0044599999999999996","p50_ttft":"0.39544999999999997","p90_e2el":12.78131,"p90_tpot":0.00517,"p90_ttft":1.267,"mean_e2el":5.485180000000001,"mean_tpot":0.00442,"mean_ttft":0.83847,"stdev_isl":172984.0197,"stdev_osl":2093.28772,"median_qps":2.0,"p50_intvty":224.2152466368,"p90_intvty":193.0501930502,"mean_intvty":225.7336343115},{"ep":8,"hw":"Vera Rubin NVL72","tp":8,"conc":1,"date":"2026-11-03","num_gpus":8,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"1265908.1909090912","tput_per_gpu":4177.49703,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","total_requests":488,"duration_seconds":3599.93059,"input_tput_per_gpu":4151.4165,"total_input_tokens":119558490,"output_tput_per_gpu":26.08054,"total_output_tokens":751105,"avg_output_tokens_expected":2115.86885,"theoretical_cache_hit_rate":0.98194,"stdev_output_tokens_expected":2537.93454,"avg_isl":244996.90574,"avg_osl":1539.14959,"p50_itl":0.0009699999999999999,"p75_isl":400632.5,"p75_osl":2121.25,"p75_qps":0.0,"p90_isl":572132.6,"p90_itl":0.00101,"p90_osl":4655.7,"p90_qps":1.0,"p95_isl":634806.65,"p95_osl":5950.15,"p95_qps":1.0,"std_qps":0.36642,"mean_itl":0.0009699999999999999,"mean_qps":0.13535,"p50_e2el":0.76022,"p50_tpot":0.0009699999999999999,"p50_ttft":0.14045,"p90_e2el":4.825060000000001,"p90_tpot":0.00101,"p90_ttft":"0.27438999999999997","mean_e2el":"1.7439500000000001","mean_tpot":0.0009699999999999999,"mean_ttft":0.2422,"stdev_isl":210269.22687,"stdev_osl":2295.75252,"median_qps":0.0,"p50_intvty":1030.9278350515,"p90_intvty":990.099009901,"mean_intvty":1030.9278350515},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":96,"date":"2026-11-03","num_gpus":4,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":41711251.2969697,"tput_per_gpu":137647.12928,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","total_requests":14804,"duration_seconds":3628.15997,"input_tput_per_gpu":136589.52376,"total_input_tokens":1982936683,"output_tput_per_gpu":1057.60552,"total_output_tokens":15381265,"avg_output_tokens_expected":1256.46398,"theoretical_cache_hit_rate":0.97049,"stdev_output_tokens_expected":2358.52779,"avg_isl":133901.28128,"avg_osl":1036.7906,"p50_itl":0.008039999999999999,"p75_isl":173963.0,"p75_osl":1076.25,"p75_qps":5.0,"p90_isl":289250.1,"p90_itl":0.00936,"p90_osl":2321.4,"p90_qps":7.0,"p95_isl":445513.2,"p95_osl":3870.1,"p95_qps":8.0,"std_qps":2.25409,"mean_itl":"0.0077599999999999995","mean_qps":4.08021,"p50_e2el":4.70131,"p50_tpot":0.008039999999999999,"p50_ttft":0.38817,"p90_e2el":"20.169439999999998","p90_tpot":0.009349999999999999,"p90_ttft":"2.9708699999999997","mean_e2el":9.31319,"mean_tpot":0.00775,"mean_ttft":1.26683,"stdev_isl":145513.12948,"stdev_osl":1986.21829,"median_qps":4.0,"p50_intvty":124.3781094527,"p90_intvty":106.8376068376,"mean_intvty":128.8659793814},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":24,"date":"2026-11-03","num_gpus":4,"scenario":"agentic","framework":"Dynamo TensorRT LLM","precision":"FP4","tput_per_mw":"13443524.754545456","tput_per_gpu":44363.63169,"model_display":"Muse-Spark-Open","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1793692800,"serving_config":"agg","total_requests":3819,"duration_seconds":3606.97318,"input_tput_per_gpu":44096.48429,"total_input_tokens":636219344,"output_tput_per_gpu":267.1474,"total_output_tokens":3854374,"avg_output_tokens_expected":1509.90556,"theoretical_cache_hit_rate":0.979,"stdev_output_tokens_expected":2261.21719,"avg_isl":166593.17727,"avg_osl":1009.26263,"p50_itl":0.00232,"p75_isl":220814.0,"p75_osl":1040.5,"p75_qps":2.0,"p90_isl":334264.6,"p90_itl":0.00256,"p90_osl":2625.4,"p90_qps":2.0,"p95_isl":434031.5,"p95_osl":4225.2,"p95_qps":3.0,"std_qps":1.0391,"mean_itl":0.00227,"mean_qps":1.05879,"p50_e2el":3.61541,"p50_tpot":0.00232,"p50_ttft":1.19275,"p90_e2el":12.31178,"p90_tpot":0.00256,"p90_ttft":8.1166,"mean_e2el":"5.4380299999999995","mean_tpot":0.00227,"mean_ttft":3.09189,"stdev_isl":145463.77339,"stdev_osl":1766.48633,"median_qps":1.0,"p50_intvty":431.0344827586,"p90_intvty":390.625,"mean_intvty":440.5286343612},{"ep":1,"hw":"Vera Rubin NVL72","tp":1,"conc":336,"date":"2026-11-05","num_gpus":4,"scenario":"agentic","decode_ep":1,"decode_tp":1,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":96330392.07878788,"tput_per_gpu":317890.29386,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"num_decode_gpu":2,"serving_config":"disagg","total_requests":47692,"num_prefill_gpu":2,"duration_seconds":3629.50319,"input_tput_per_gpu":629676.66706,"total_input_tokens":9158941658,"output_tput_per_gpu":6103.92066,"total_output_tokens":44476499,"avg_output_tokens_expected":1290.5117,"theoretical_cache_hit_rate":0.95715,"stdev_output_tokens_expected":2430.34647,"avg_isl":95840.53799,"avg_osl":929.05307,"p50_itl":0.00664,"p75_isl":139104.75,"p75_osl":933.0,"p75_qps":16.0,"p90_isl":193524.7,"p90_itl":0.00736,"p90_osl":2086.9,"p90_qps":19.0,"p95_isl":221061.5,"p95_osl":3512.35,"p95_qps":21.0,"std_qps":4.47149,"mean_itl":0.00723,"mean_qps":13.18132,"p50_e2el":6.07203,"p50_tpot":0.00662,"p50_ttft":"2.0576399999999997","p90_e2el":33.80232,"p90_tpot":0.00732,"p90_ttft":11.01609,"mean_e2el":"14.469719999999999","mean_tpot":0.00721,"mean_ttft":8.465770000000001,"stdev_isl":64289.2707,"stdev_osl":1814.3293,"median_qps":13.0,"p50_intvty":150.6024096386,"p90_intvty":135.8695652174,"mean_intvty":138.3125864454},{"ep":1,"hw":"Vera Rubin NVL72","tp":1,"conc":176,"date":"2026-11-05","num_gpus":3,"scenario":"agentic","decode_ep":1,"decode_tp":1,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":68994174.73030303,"tput_per_gpu":227680.77661,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"num_decode_gpu":2,"serving_config":"disagg","total_requests":24718,"num_prefill_gpu":1,"duration_seconds":3629.2353,"input_tput_per_gpu":676744.22481,"total_input_tokens":4915723165,"output_tput_per_gpu":3149.05251,"total_output_tokens":22917533,"avg_output_tokens_expected":1271.15595,"theoretical_cache_hit_rate":0.95843,"stdev_output_tokens_expected":2337.72915,"avg_isl":99363.38001,"avg_osl":924.72308,"p50_itl":0.00417,"p75_isl":142815.5,"p75_osl":909.75,"p75_qps":9.0,"p90_isl":199267.6,"p90_itl":"0.0048200000000000005","p90_osl":2038.3,"p90_qps":11.0,"p95_isl":224989.7,"p95_osl":3493.75,"p95_qps":12.0,"std_qps":2.86775,"mean_itl":0.00475,"mean_qps":6.81257,"p50_e2el":"3.0450999999999997","p50_tpot":0.00417,"p50_ttft":0.93368,"p90_e2el":22.37418,"p90_tpot":"0.0048200000000000005","p90_ttft":5.21209,"mean_e2el":13.47634,"mean_tpot":0.00475,"mean_ttft":9.688049999999999,"stdev_isl":64823.68498,"stdev_osl":1896.24784,"median_qps":7.0,"p50_intvty":239.8081534772,"p90_intvty":207.468879668,"mean_intvty":210.5263157895},{"ep":1,"hw":"GB300 NVL72","tp":1,"conc":36,"date":"2026-11-06","num_gpus":1,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":69980153.51415095,"tput_per_gpu":148357.92545,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42124076518/attempts/5","total_requests":4960,"duration_seconds":3623.19823,"input_tput_per_gpu":147026.65962,"total_input_tokens":532706733,"output_tput_per_gpu":1331.26583,"total_output_tokens":4823440,"avg_output_tokens_expected":1332.03465,"theoretical_cache_hit_rate":0.96387,"stdev_output_tokens_expected":2100.21223,"avg_isl":107400.55101,"avg_osl":972.46774,"p50_itl":0.0073,"p75_isl":151904.0,"p75_osl":996.25,"p75_qps":2.0,"p90_isl":210620.8,"p90_itl":0.00932,"p90_osl":2336.3,"p90_qps":3.0,"p95_isl":235168.75,"p95_osl":3783.35,"p95_qps":3.0,"std_qps":1.11599,"mean_itl":0.00785,"mean_qps":1.37065,"p50_e2el":5.16778,"p50_tpot":0.00729,"p50_ttft":1.22071,"p90_e2el":27.07932,"p90_tpot":0.00932,"p90_ttft":6.58829,"mean_e2el":12.34035,"mean_tpot":0.00785,"mean_ttft":4.739520000000001,"stdev_isl":65624.11292,"stdev_osl":2140.77976,"median_qps":1.0,"p50_intvty":136.9863013699,"p90_intvty":107.2961373391,"mean_intvty":127.3885350318},{"ep":1,"hw":"GB300 NVL72","tp":1,"conc":80,"date":"2026-11-06","num_gpus":1,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"109729171.58962265","tput_per_gpu":232625.84377,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42124076518/attempts/5","total_requests":10099,"duration_seconds":3628.12263,"input_tput_per_gpu":230069.66914,"total_input_tokens":834720972,"output_tput_per_gpu":2556.17463,"total_output_tokens":9274115,"avg_output_tokens_expected":1191.38067,"theoretical_cache_hit_rate":0.94757,"stdev_output_tokens_expected":2198.38411,"avg_isl":82653.82434,"avg_osl":918.32013,"p50_itl":0.02249,"p75_isl":118792.0,"p75_osl":1008.5,"p75_qps":4.0,"p90_isl":179438.2,"p90_itl":"0.028489999999999998","p90_osl":1970.2,"p90_qps":5.0,"p95_isl":216845.9,"p95_osl":3388.0,"p95_qps":6.0,"std_qps":1.76895,"mean_itl":0.02246,"mean_qps":2.7895,"p50_e2el":13.86957,"p50_tpot":0.02249,"p50_ttft":2.4347,"p90_e2el":"49.946239999999996","p90_tpot":"0.028489999999999998","p90_ttft":8.08701,"mean_e2el":24.49542,"mean_tpot":0.02246,"mean_ttft":"3.5575900000000003","stdev_isl":63225.70834,"stdev_osl":1779.53647,"median_qps":3.0,"p50_intvty":44.4642063139,"p90_intvty":35.1000351,"mean_intvty":44.5235975067},{"ep":1,"hw":"GB300 NVL72","tp":1,"conc":56,"date":"2026-11-06","num_gpus":1,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":90510661.25471698,"tput_per_gpu":191882.60186,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42124076518/attempts/5","total_requests":7773,"duration_seconds":3628.06693,"input_tput_per_gpu":189884.92427,"total_input_tokens":689231488,"output_tput_per_gpu":1997.67759,"total_output_tokens":7267719,"avg_output_tokens_expected":1188.78961,"theoretical_cache_hit_rate":0.95503,"stdev_output_tokens_expected":2021.98785,"avg_isl":88629.25691,"avg_osl":932.42094,"p50_itl":"0.013460000000000001","p75_isl":123833.0,"p75_osl":981.0,"p75_qps":3.0,"p90_isl":193810.4,"p90_itl":0.01697,"p90_osl":2145.6,"p90_qps":4.0,"p95_isl":228039.0,"p95_osl":3525.2,"p95_qps":5.0,"std_qps":1.53528,"mean_itl":"0.013789999999999998","mean_qps":2.14787,"p50_e2el":9.05,"p50_tpot":"0.013460000000000001","p50_ttft":"1.5366099999999998","p90_e2el":33.96089,"p90_tpot":0.01697,"p90_ttft":7.6235,"mean_e2el":15.98126,"mean_tpot":"0.013779999999999999","mean_ttft":2.98787,"stdev_isl":64433.22743,"stdev_osl":1870.63601,"median_qps":2.0,"p50_intvty":74.294205052,"p90_intvty":58.9275191514,"mean_intvty":72.5163161711},{"ep":1,"hw":"Vera Rubin NVL72","tp":1,"conc":10,"date":"2026-11-05","num_gpus":1,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":20247704.97878788,"tput_per_gpu":66817.42643,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"serving_config":"agg","total_requests":2290,"duration_seconds":3604.71619,"input_tput_per_gpu":66194.44063,"total_input_tokens":238612172,"output_tput_per_gpu":622.9858,"total_output_tokens":2245687,"avg_output_tokens_expected":1732.82153,"theoretical_cache_hit_rate":0.96196,"stdev_output_tokens_expected":2621.32752,"avg_isl":104197.45502,"avg_osl":980.64934,"p50_itl":"0.0019199999999999998","p75_isl":134181.0,"p75_osl":884.0,"p75_qps":1.0,"p90_isl":187264.8,"p90_itl":"0.0024500000000000004","p90_osl":2258.4,"p90_qps":2.0,"p95_isl":217583.7,"p95_osl":4315.45,"p95_qps":2.0,"std_qps":0.84808,"mean_itl":"0.0020099999999999996","mean_qps":0.63566,"p50_e2el":"1.0673499999999998","p50_tpot":0.00191,"p50_ttft":0.15514,"p90_e2el":5.03095,"p90_tpot":"0.0024500000000000004","p90_ttft":0.84245,"mean_e2el":"2.3672600000000004","mean_tpot":"0.0020099999999999996","mean_ttft":0.38091,"stdev_isl":55625.16361,"stdev_osl":2050.93232,"median_qps":0.0,"p50_intvty":520.8333333333,"p90_intvty":408.1632653061,"mean_intvty":497.5124378109},{"ep":1,"hw":"Vera Rubin NVL72","tp":1,"conc":120,"date":"2026-11-05","num_gpus":1,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"142742298.84848484","tput_per_gpu":471049.5862,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"serving_config":"agg","total_requests":18319,"duration_seconds":3620.68195,"input_tput_per_gpu":466265.59902,"total_input_tokens":1688587013,"output_tput_per_gpu":4783.98718,"total_output_tokens":17346396,"avg_output_tokens_expected":1249.32125,"theoretical_cache_hit_rate":0.9562,"stdev_output_tokens_expected":2302.495,"avg_isl":92155.65457,"avg_osl":945.5372,"p50_itl":0.01114,"p75_isl":135310.0,"p75_osl":977.5,"p75_qps":7.0,"p90_isl":191807.6,"p90_itl":0.01427,"p90_osl":2084.0,"p90_qps":9.0,"p95_isl":222126.6,"p95_osl":3532.1,"p95_qps":10.0,"std_qps":2.664,"mean_itl":0.01099,"mean_qps":5.06582,"p50_e2el":5.98361,"p50_tpot":"0.011130000000000001","p50_ttft":0.64871,"p90_e2el":28.40706,"p90_tpot":0.01426,"p90_ttft":3.85377,"mean_e2el":"12.468860000000001","mean_tpot":"0.010960000000000001","mean_ttft":2.04789,"stdev_isl":65257.32412,"stdev_osl":1832.49107,"median_qps":5.0,"p50_intvty":89.7666068223,"p90_intvty":70.0770847933,"mean_intvty":90.991810737},{"ep":1,"hw":"Vera Rubin NVL72","tp":1,"conc":13,"date":"2026-11-05","num_gpus":1,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"25716378.924242422","tput_per_gpu":84864.05045,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"serving_config":"agg","total_requests":2889,"duration_seconds":3603.06897,"input_tput_per_gpu":84140.60551,"total_input_tokens":303164405,"output_tput_per_gpu":723.44494,"total_output_tokens":2606622,"avg_output_tokens_expected":1612.91741,"theoretical_cache_hit_rate":0.96248,"stdev_output_tokens_expected":2485.81692,"avg_isl":104937.48875,"avg_osl":902.25753,"p50_itl":0.00196,"p75_isl":141303.0,"p75_osl":789.0,"p75_qps":1.0,"p90_isl":191283.2,"p90_itl":0.00226,"p90_osl":2079.2,"p90_qps":2.0,"p95_isl":219741.2,"p95_osl":3912.0,"p95_qps":3.0,"std_qps":0.93244,"mean_itl":0.002,"mean_qps":0.80217,"p50_e2el":1.11558,"p50_tpot":0.00195,"p50_ttft":0.20775,"p90_e2el":5.72499,"p90_tpot":0.00225,"p90_ttft":1.20068,"mean_e2el":"2.4826599999999996","mean_tpot":0.00198,"mean_ttft":0.65322,"stdev_isl":58158.7021,"stdev_osl":1924.27778,"median_qps":1.0,"p50_intvty":510.2040816327,"p90_intvty":442.4778761062,"mean_intvty":500.0},{"ep":1,"hw":"Vera Rubin NVL72","tp":4,"conc":1,"date":"2026-11-05","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"1854503.4727272727","tput_per_gpu":6119.86146,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"serving_config":"agg","total_requests":936,"duration_seconds":3598.3701,"input_tput_per_gpu":6062.14748,"total_input_tokens":87255401,"output_tput_per_gpu":57.71398,"total_output_tokens":830705,"avg_output_tokens_expected":1594.32265,"theoretical_cache_hit_rate":0.95414,"stdev_output_tokens_expected":2435.26488,"avg_isl":93221.58226,"avg_osl":887.50534,"p50_itl":0.00138,"p75_isl":126341.5,"p75_osl":688.75,"p75_qps":0.0,"p90_isl":193096.5,"p90_itl":"0.0014199999999999998","p90_osl":2402.5,"p90_qps":1.0,"p95_isl":226450.5,"p95_osl":4248.25,"p95_qps":1.0,"std_qps":0.48796,"mean_itl":0.00138,"mean_qps":0.26016,"p50_e2el":0.55875,"p50_tpot":0.00138,"p50_ttft":0.15409,"p90_e2el":4.44796,"p90_tpot":0.00141,"p90_ttft":"0.40826999999999997","mean_e2el":1.58138,"mean_tpot":"0.0013700000000000001","mean_ttft":0.35847,"stdev_isl":61291.76998,"stdev_osl":1908.03353,"median_qps":0.0,"p50_intvty":724.6376811594,"p90_intvty":704.2253521127,"mean_intvty":724.6376811594},{"ep":1,"hw":"Vera Rubin NVL72","tp":1,"conc":22,"date":"2026-11-05","num_gpus":1,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":40840575.63636364,"tput_per_gpu":134773.8996,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793865600,"serving_config":"agg","total_requests":4350,"duration_seconds":3601.16561,"input_tput_per_gpu":133674.86489,"total_input_tokens":481385327,"output_tput_per_gpu":1099.03471,"total_output_tokens":3957806,"avg_output_tokens_expected":1371.62273,"theoretical_cache_hit_rate":0.96568,"stdev_output_tokens_expected":2087.62854,"avg_isl":110663.29356,"avg_osl":909.84046,"p50_itl":0.00226,"p75_isl":157331.5,"p75_osl":953.5,"p75_qps":2.0,"p90_isl":211111.3,"p90_itl":0.00282,"p90_osl":2316.7,"p90_qps":3.0,"p95_isl":232056.85,"p95_osl":3742.7,"p95_qps":3.0,"std_qps":1.17318,"mean_itl":0.00232,"mean_qps":1.2094,"p50_e2el":1.48226,"p50_tpot":"0.0022400000000000002","p50_ttft":0.46168,"p90_e2el":6.91622,"p90_tpot":0.0028,"p90_ttft":1.33473,"mean_e2el":"2.9725300000000003","mean_tpot":0.0023,"mean_ttft":0.81848,"stdev_isl":64280.95195,"stdev_osl":1580.3033,"median_qps":1.0,"p50_intvty":442.4778761062,"p90_intvty":354.609929078,"mean_intvty":431.0344827586},{"ep":4,"hw":"GB300 NVL72","tp":4,"conc":3,"date":"2026-11-06","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"2462223.0566037735","tput_per_gpu":5219.91288,"model_display":"Qwen 3.8 Flash Next","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1793952000,"serving_config":"agg","source_run_url":"https://github.com/SemiAnalysisAI/InferenceX/actions/runs/42124076518/attempts/5","total_requests":802,"duration_seconds":3598.36514,"input_tput_per_gpu":5162.09759,"total_input_tokens":74300448,"output_tput_per_gpu":57.81528,"total_output_tokens":832162,"avg_output_tokens_expected":1638.50623,"theoretical_cache_hit_rate":0.95065,"stdev_output_tokens_expected":2643.33212,"avg_isl":92643.95012,"avg_osl":1037.60848,"p50_itl":0.00239,"p75_isl":130052.0,"p75_osl":830.5,"p75_qps":0.0,"p90_isl":203933.8,"p90_itl":0.00254,"p90_osl":2718.4,"p90_qps":1.0,"p95_isl":231492.7,"p95_osl":5220.05,"p95_qps":1.0,"std_qps":0.46963,"mean_itl":0.00242,"mean_qps":0.22293,"p50_e2el":1.45959,"p50_tpot":0.00239,"p50_ttft":0.3491,"p90_e2el":"11.821959999999999","p90_tpot":0.00254,"p90_ttft":5.03801,"mean_e2el":"4.3128400000000005","mean_tpot":"0.0024100000000000002","mean_ttft":"1.7753299999999999","stdev_isl":65422.23512,"stdev_osl":2152.7154,"median_qps":0.0,"p50_intvty":418.410041841,"p90_intvty":393.7007874016,"mean_intvty":413.2231404959},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8192,"date":"2026-06-15","num_gpus":16,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5587543.710053067,"tput_per_gpu":11845.5926653125,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":11849.9939021185,"output_tput_per_gpu":11841.1914285065,"p50_itl":14.513413675013,"mean_itl":12.5502723431211,"p50_e2el":76.0489374310127,"p50_tpot":0.07699958096899999,"p50_ttft":2.2581479385262,"mean_e2el":77.5894460005816,"mean_tpot":0.0732407522266,"mean_ttft":10.177539545207,"p50_intvty":12.9870836622,"mean_intvty":13.6536008929},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":15,"date":"2026-06-15","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":64371.27680094339,"tput_per_gpu":136.467106818,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":412.7232333795,"output_tput_per_gpu":81.2158815057,"p50_itl":1.5179410870187,"mean_itl":1.3190271338342,"p50_e2el":8.0136697939597,"p50_tpot":0.0054536146498,"p50_ttft":2.6764435659861,"mean_e2el":8.0119803651391,"mean_tpot":"0.0057365783884999995","mean_ttft":2.779839591013,"p50_intvty":183.3646240541,"mean_intvty":174.3199399142},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":8609,"date":"2026-01-28","num_gpus":26,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":7257145.211366736,"tput_per_gpu":13788.5759015968,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":10,"input_tput_per_gpu":17928.2711817752,"output_tput_per_gpu":11201.2663514853,"p50_itl":7.5244735160377,"mean_itl":6.7733019084734,"p50_e2el":41.0306475614198,"p50_tpot":"0.042322176760799995","p50_ttft":2.122890813509,"mean_e2el":42.6891213911476,"mean_tpot":0.0413759694217,"mean_ttft":"4.6137865847534005","p50_intvty":23.6282742651,"mean_intvty":24.1686180161},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":48,"date":"2026-03-23","num_gpus":56,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":324737.0684863504,"tput_per_gpu":444.8897838263,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":48,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2766.8533360698,"output_tput_per_gpu":57.8958584523,"p50_itl":2.1325586325256,"mean_itl":1.9446091976934,"p50_e2el":15.5658714940073,"p50_tpot":0.009120842769699999,"p50_ttft":7.1218101484701,"mean_e2el":15.2945382871068,"mean_tpot":0.0087040239289,"mean_ttft":7.240615400263501,"p50_intvty":109.6389911825,"mean_intvty":114.8893900296},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-03-23","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1152917.8308645987","tput_per_gpu":1579.4974282845,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":3948.6272878215,"output_tput_per_gpu":987.2149634002,"p50_itl":5.7791159139015,"mean_itl":5.1160261801011,"p50_e2el":27.8011706279358,"p50_tpot":"0.024539258218299997","p50_ttft":5.098565860535,"mean_e2el":28.5420249824912,"mean_tpot":"0.023912256876099998","mean_ttft":6.529623042377899,"p50_intvty":40.7510280507,"mean_intvty":41.819557442},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":12926,"date":"2026-01-28","num_gpus":26,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"7671806.9533240525","tput_per_gpu":14576.4332113157,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":10,"input_tput_per_gpu":18950.892090478,"output_tput_per_gpu":11842.3964118392,"p50_itl":10.640068282024,"mean_itl":9.525670334418301,"p50_e2el":58.4516322450945,"p50_tpot":0.0599475257789,"p50_ttft":3.4536425890401,"mean_e2el":60.7731927356887,"mean_tpot":0.0581497361196,"mean_ttft":"7.2595010288409005","p50_intvty":16.6812555982,"mean_intvty":17.1969825958},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":12370212.81871345,"tput_per_gpu":21153.06392,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":55.94816,"input_tput_per_gpu":10628.45,"kv_cache_precision":"FP8","total_input_tokens":1189284,"output_tput_per_gpu":10524.62,"total_output_tokens":1177666,"tput_requests_per_sec":22.87832,"p50_e2el":5.81823,"p50_tpot":"0.0056289999999999995","p50_ttft":0.059314,"p50_intvty":177.6376},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"1275137.1608187135","tput_per_gpu":2180.484545,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":17.01858,"input_tput_per_gpu":1091.704,"kv_cache_precision":"FP8","total_input_tokens":74317,"output_tput_per_gpu":1088.781,"total_output_tokens":74118,"tput_requests_per_sec":4.700745,"p50_e2el":"1.7503119999999999","p50_tpot":0.001674,"p50_ttft":"0.037753999999999996","p50_intvty":597.3522},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"2336776.0953216376","tput_per_gpu":3995.887123,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":23.14442,"input_tput_per_gpu":2008.194,"kv_cache_precision":"FP8","total_input_tokens":92957,"output_tput_per_gpu":1987.693,"total_output_tokens":92008,"tput_requests_per_sec":4.320695,"p50_e2el":2.36565,"p50_tpot":0.002274,"p50_ttft":"0.039445999999999995","p50_intvty":439.7722},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":207789.9680613869,"tput_per_gpu":284.6722562441,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":252.8707047787,"output_tput_per_gpu":31.8015514655,"p50_itl":0.007609709166,"p90_itl":0.0077086752281,"mean_itl":0.0076158869826,"p50_e2el":7.1973518542945,"p50_tpot":0.0075993114011,"p50_ttft":0.2245859559625,"p90_e2el":7.8399149456061,"p90_tpot":0.0076702428642,"p90_ttft":0.2725449006073,"mean_e2el":7.312617384176701,"mean_tpot":0.0076139910521,"mean_ttft":0.2366970940493,"p50_intvty":131.5908701746,"p90_intvty":130.3739682966,"mean_intvty":131.3371651157},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-03","num_gpus":20,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1441325.4519222172","tput_per_gpu":3055.6099580751,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13580.1441432278,"output_tput_per_gpu":424.476411787,"p50_itl":0.7970074745826,"mean_itl":1.4635452816841,"p50_e2el":"32.361331379041104","p50_tpot":0.0073370727266,"p50_ttft":24.5801331936382,"mean_e2el":31.7068590470133,"mean_tpot":0.0084445143004,"mean_ttft":23.9340616291867,"p50_intvty":136.2941376294,"mean_intvty":118.4200730118},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-06-03","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":4,"tput_per_mw":91214.42909212265,"tput_per_gpu":193.3745896753,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":343.5495105528,"output_tput_per_gpu":43.1996687979,"p50_itl":0.0156661910005,"mean_itl":0.0156938181042,"p50_e2el":5.056824533036,"p50_tpot":"0.0040869804479000005","p50_ttft":1.0243052095175,"mean_e2el":5.3824310713913,"mean_tpot":0.004300664597,"mean_ttft":1.3710916855838,"p50_intvty":244.6794186442,"mean_intvty":232.5222014983},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"154526.17151966982","tput_per_gpu":327.5954836217,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":2035.4088784718,"output_tput_per_gpu":42.9599178133,"p50_itl":0.0172008594964,"mean_itl":0.0173189096862,"p50_e2el":6.1888493159786,"p50_tpot":0.004934216506500001,"p50_ttft":1.2741082410794,"mean_e2el":6.8101945686154,"mean_tpot":0.0055208793165,"mean_ttft":1.6989514714922,"p50_intvty":202.6664210368,"mean_intvty":181.1305668292},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":654,"date":"2026-01-28","num_gpus":10,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":3844119.452105053,"tput_per_gpu":7303.8269589996,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":18264.315941759,"output_tput_per_gpu":4563.7047133098,"p50_itl":3.2154342789436,"mean_itl":"2.8258965525304003","p50_e2el":"15.862855603103501","p50_tpot":0.0135885171192,"p50_ttft":3.3765829774784,"mean_e2el":15.7156265550798,"mean_tpot":"0.012986853768699999","mean_ttft":3.7682201236046,"p50_intvty":73.5915472769,"mean_intvty":77.0009440168},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":352,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":294165.8698764964,"tput_per_gpu":403.0072417308,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2416.3864516296,"output_tput_per_gpu":219.9727681037,"p50_itl":3.7587406649254,"mean_itl":3.3297293832575,"p50_e2el":16.0249405568466,"p50_tpot":0.015974612093,"p50_ttft":"1.2521666130051001","mean_e2el":16.1018847523605,"mean_tpot":"0.015568193962200001","mean_ttft":"1.7717513356321999","p50_intvty":62.5993291215,"mean_intvty":64.2335265368},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":826534.7093567251,"tput_per_gpu":1413.374353,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":13.00363,"input_tput_per_gpu":705.2645,"kv_cache_precision":"FP8","total_input_tokens":9171,"output_tput_per_gpu":708.1099,"total_output_tokens":9208,"tput_requests_per_sec":0.769016,"p50_e2el":1.196685,"p50_tpot":0.001142,"p50_ttft":0.028784,"p50_intvty":875.9301},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"26465229.292397663","tput_per_gpu":45255.54209,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":117.6293,"input_tput_per_gpu":40241.05,"kv_cache_precision":"FP8","total_input_tokens":4733525,"output_tput_per_gpu":5014.492,"total_output_tokens":589851,"tput_requests_per_sec":5.440823,"p50_e2el":12.62134,"p50_tpot":0.012163,"p50_ttft":0.178562,"p50_intvty":82.21638},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":256,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":91094.33962264152,"tput_per_gpu":193.12,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":105.3741},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1568566.2944001462","tput_per_gpu":2148.9358233282,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":1910.8199869818,"output_tput_per_gpu":238.1158363464,"p50_itl":"0.013352463487499999","p90_itl":0.0141692050733,"mean_itl":0.0158185986026,"p50_e2el":"15.020055574365001","p50_tpot":0.0159954606943,"p50_ttft":0.2520945505239,"p90_e2el":"16.659811503533298","p90_tpot":"0.016420186971500002","p90_ttft":0.6127600804903,"mean_e2el":14.9569875109009,"mean_tpot":0.0158037249625,"mean_ttft":"0.49760214287090004","p50_intvty":62.517736695,"p90_intvty":60.9006463651,"mean_intvty":63.2762214208},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":666,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1047018.1896746226","tput_per_gpu":2219.6785621102,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":10002.7126557868,"output_tput_per_gpu":1246.7993004007,"p50_itl":1.1976373725338,"mean_itl":"1.1454662726773999","p50_e2el":13.717044361867,"p50_tpot":0.0120707270371,"p50_ttft":2.2639850485139,"mean_e2el":"14.584715410713398","mean_tpot":0.0120898283328,"mean_ttft":3.4533976601316,"p50_intvty":82.8450512488,"mean_intvty":82.7141604057},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":10,"date":"2026-06-15","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":37084.4155265566,"tput_per_gpu":78.6189609163,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":236.9768571052,"output_tput_per_gpu":46.9473816786,"p50_itl":1.3340574320173,"mean_itl":1.1577961430254,"p50_e2el":9.5337457649293,"p50_tpot":0.0047630198337,"p50_ttft":4.9097944915411,"mean_e2el":9.214984409968599,"mean_tpot":0.0049420256521,"mean_ttft":4.6880016868922,"p50_intvty":209.9508368467,"mean_intvty":202.3461775391},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-06-03","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":3440833.134963066,"tput_per_gpu":7294.5662461217,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":12968.2340495524,"output_tput_per_gpu":1620.898442691,"p50_itl":"1.3160747655201999","mean_itl":2.4050855283754,"p50_e2el":32.0834125680849,"p50_tpot":"0.011481770045399999","p50_ttft":"19.633636169834098","mean_e2el":33.0487133824372,"mean_tpot":0.0130451591652,"mean_ttft":"21.035499178599302","p50_intvty":87.0945852462,"mean_intvty":76.6567879577},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":754412.1020806604,"tput_per_gpu":1599.353656411,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":9953.5847559571,"output_tput_per_gpu":206.9818064866,"p50_itl":0.0237789065577,"mean_itl":"0.024056054856900003","p50_e2el":10.5647596861236,"p50_tpot":0.0068259806862,"p50_ttft":3.5513929389417,"mean_e2el":11.1216061303021,"mean_tpot":0.007537346633200001,"mean_ttft":"4.1854186884177995","p50_intvty":146.499096023,"mean_intvty":132.6726829305},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-03-23","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1511295.0785126279","tput_per_gpu":2070.4742575623,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":2453.8331570085,"output_tput_per_gpu":920.3975592235,"p50_itl":7.128685398958599,"mean_itl":6.2615932894786,"p50_e2el":"30.390580075560102","p50_tpot":"0.031841018426800004","p50_ttft":"1.0001772900577999","mean_e2el":30.6716454043935,"mean_tpot":0.0306193466795,"mean_ttft":"2.4773543244737004","p50_intvty":31.4060306299,"mean_intvty":32.6590900344},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":44,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":67995.0105530657,"tput_per_gpu":93.1531644577,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":557.220880693,"output_tput_per_gpu":50.9651902545,"p50_itl":"2.0108163801487997","mean_itl":1.7882222674759,"p50_e2el":8.573992217425301,"p50_tpot":0.008567553439699999,"p50_ttft":"0.28457346255890004","mean_e2el":8.709044318647301,"mean_tpot":0.0083665543895,"mean_ttft":0.9668549947901001,"p50_intvty":116.7194353714,"mean_intvty":119.5235163062},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":128,"date":"2026-03-23","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":1331580.96543073,"tput_per_gpu":1824.2659226401,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":2433.3367769047,"output_tput_per_gpu":606.124214111,"p50_itl":5.3531723900232,"mean_itl":4.7118089012799,"p50_e2el":23.0621167789213,"p50_tpot":0.0238648674936,"p50_ttft":1.0732285425765,"mean_e2el":23.2944489746889,"mean_tpot":"0.023043236521299998","mean_ttft":2.1170768351476,"p50_intvty":41.9026001409,"mean_intvty":43.3966816717},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":13,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":27076.34636160584,"tput_per_gpu":37.0945945154,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":224.0171466158,"output_tput_per_gpu":20.1016352336,"p50_itl":"1.3468476959970002","mean_itl":1.2083276187869,"p50_e2el":6.154064772534199,"p50_tpot":0.0057200187123,"p50_ttft":0.7743423988577,"mean_e2el":6.3153375526836,"mean_tpot":0.0055470999034,"mean_ttft":1.2496566695322,"p50_intvty":174.8246029086,"mean_intvty":180.2743807424},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":4433306.594195986,"tput_per_gpu":6073.6300340485,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":5398.5881676152,"output_tput_per_gpu":675.0418664333,"p50_itl":0.0502471146174,"p90_itl":0.2216530149803,"mean_itl":0.09081530394819999,"p50_e2el":86.7299884571694,"p50_tpot":0.0929712108878,"p50_ttft":0.6866031568497,"p90_e2el":99.90428454484791,"p90_tpot":0.0978414058588,"p90_ttft":1.6750296341255,"mean_e2el":86.41183020444879,"mean_tpot":0.0906308234926,"mean_ttft":2.7661818072014,"p50_intvty":10.7560178086,"p90_intvty":10.2206217421,"mean_intvty":11.0337737368},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":21174.06645334906,"tput_per_gpu":44.8890208811,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":201.3612609878,"output_tput_per_gpu":25.3299908678,"p50_itl":"1.2762638080167001","mean_itl":1.1033146499322,"p50_e2el":7.9454655584996,"p50_tpot":0.0044815368053,"p50_ttft":3.9221377320355,"mean_e2el":8.7067804407649,"mean_tpot":0.0046991139131,"mean_ttft":"4.3486877947317994","p50_intvty":223.1377412372,"mean_intvty":212.8060775902},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":60,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":175749.0638132547,"tput_per_gpu":372.5880152841,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":1118.8431964769,"output_tput_per_gpu":223.3369790456,"p50_itl":0.8769973290619,"mean_itl":0.8295367688758,"p50_e2el":"11.702219558996099","p50_tpot":0.008786612288,"p50_ttft":3.6655160235241,"mean_e2el":11.9723326866185,"mean_tpot":0.0087677160087,"mean_ttft":3.8912621451243,"p50_intvty":113.8095055554,"mean_intvty":114.0547890708},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":333,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":559245.1055881132,"tput_per_gpu":1185.5996238467999,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":5337.9573304585,"output_tput_per_gpu":666.5549105203,"p50_itl":1.1348010299989,"mean_itl":1.0809189919059,"p50_e2el":12.7463179579936,"p50_tpot":0.0114036693098,"p50_ttft":"1.9344247580010998","mean_e2el":13.6788629710231,"mean_tpot":0.0114167546954,"mean_ttft":"3.1579182523141998","p50_intvty":87.6910731826,"mean_intvty":87.5905655049},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":24,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":65066.29948650944,"tput_per_gpu":137.9405549114,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":622.4512075455,"output_tput_per_gpu":77.3767233321,"p50_itl":0.628494206816,"mean_itl":0.5977060778618001,"p50_e2el":8.2787137804553,"p50_tpot":0.0062961911547,"p50_ttft":2.4169629674871,"mean_e2el":8.5888644875444,"mean_tpot":"0.0063119671970000004","mean_ttft":"2.7985869907842003","p50_intvty":158.8261816435,"mean_intvty":158.4292137129},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-23","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1088177.4229932118","tput_per_gpu":1490.8030695007,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2650.8627670103,"output_tput_per_gpu":330.743371991,"p50_itl":"4.2325079796136995","mean_itl":3.7117418541156,"p50_e2el":"20.792238746071202","p50_tpot":0.0190165639352,"p50_ttft":"3.4724854715167996","mean_e2el":20.9988508990788,"mean_tpot":0.0183384768399,"mean_ttft":4.1220249922619,"p50_intvty":52.585735436,"mean_intvty":54.5301558428},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":144,"date":"2026-03-23","num_gpus":80,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"179663.42858649636","tput_per_gpu":246.1388971635,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":72,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":1232.1161136432,"output_tput_per_gpu":136.5858731102,"p50_itl":3.0903065650491,"mean_itl":2.7789475023307,"p50_e2el":12.873184332042,"p50_tpot":0.0132927277366,"p50_ttft":0.3139074085047,"mean_e2el":"12.925549696154299","mean_tpot":"0.012952013944799999","mean_ttft":1.033970366326,"p50_intvty":75.2291042004,"mean_intvty":77.2080700547},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":88,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"110953.09988532848","tput_per_gpu":152.0057468429,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":913.2566097678,"output_tput_per_gpu":82.8011229406,"p50_itl":"2.5276487809606003","mean_itl":2.2571328926397,"p50_e2el":10.6023437589174,"p50_tpot":0.0107832621213,"p50_ttft":0.2943238376174,"mean_e2el":10.6261599911141,"mean_tpot":0.0105348434,"mean_ttft":0.9384431952719,"p50_intvty":92.7363156667,"mean_intvty":94.9231006128},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":20099699.56140351,"tput_per_gpu":34370.48625,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":78.10109,"input_tput_per_gpu":30303.83,"kv_cache_precision":"FP8","total_input_tokens":4733525,"output_tput_per_gpu":3776.202,"total_output_tokens":589851,"tput_requests_per_sec":8.194507,"p50_e2el":8.271427,"p50_tpot":0.007989,"p50_ttft":0.098709,"p50_intvty":124.8632},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1062423.8171978833","tput_per_gpu":1455.5206295611,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":1291.8955616259,"output_tput_per_gpu":163.6250679351,"p50_itl":0.0104668010026,"p90_itl":0.0107848839834,"mean_itl":0.011641347409,"p50_e2el":"11.142788457684201","p50_tpot":"0.011645068570799999","p50_ttft":0.2414523852058,"p90_e2el":12.3230052732863,"p90_tpot":"0.012127916736599999","p90_ttft":0.5753446418792,"mean_e2el":11.1153358526179,"mean_tpot":"0.011627665454300001","mean_ttft":0.3498545389157,"p50_intvty":85.8732599054,"p90_intvty":82.4543919389,"mean_intvty":86.0017863371},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":2253,"date":"2026-06-22","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2989977.4671514155","tput_per_gpu":6338.752230361,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":15863.9866250141,"output_tput_per_gpu":3957.4436316978,"p50_itl":1.4736939379945,"mean_itl":"1.4171433772836002","p50_e2el":15.2410323570948,"p50_tpot":0.0150028991546,"p50_ttft":1.0940053289523,"mean_e2el":15.6394351068531,"mean_tpot":"0.014959815853500001","mean_ttft":1.8557188238601,"p50_intvty":66.653784025,"mean_intvty":66.8457426074},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":30,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"105093.32180787736","tput_per_gpu":222.7978422327,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":668.7314685816,"output_tput_per_gpu":133.6111169629,"p50_itl":0.720449479064,"mean_itl":0.6834093139299,"p50_e2el":9.737437851959799,"p50_tpot":0.0072113116552,"p50_ttft":3.1372722201049,"mean_e2el":9.868754026909201,"mean_tpot":"0.0072257440509000005","mean_ttft":3.2100694242244,"p50_intvty":138.671027936,"mean_intvty":138.3940522887},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":9,"date":"2026-03-23","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"112596.14075576644","tput_per_gpu":154.2567128354,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":1096.0663917866,"output_tput_per_gpu":19.7124729853,"p50_itl":1.4488349300809,"mean_itl":1.2763731882397,"p50_e2el":6.5885202679783,"p50_tpot":0.005981527008199999,"p50_ttft":"1.3340362359303999","mean_e2el":7.0340974044288,"mean_tpot":0.0055731402352,"mean_ttft":"1.9144230640492998","p50_intvty":167.1813900746,"mean_intvty":179.4320540651},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":20,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"116353.03737905264","tput_per_gpu":221.0707710202,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":2322.6602891001,"output_tput_per_gpu":115.9912951162,"p50_itl":0.9759253894444999,"mean_itl":0.870681476059,"p50_e2el":3.9051218144596,"p50_tpot":0.0041424390571,"p50_ttft":0.1181764829671,"mean_e2el":3.8018023827835,"mean_tpot":0.00395034006,"mean_ttft":0.1710609053087,"p50_intvty":241.4036721385,"mean_intvty":253.1427636104},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":118893.6941520468,"tput_per_gpu":203.308217,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":11.29996,"input_tput_per_gpu":101.4495,"kv_cache_precision":"FP8","total_input_tokens":9171,"output_tput_per_gpu":101.8588,"total_output_tokens":9208,"tput_requests_per_sec":0.884959,"p50_e2el":1.042741,"p50_tpot":0.000983,"p50_ttft":0.037488,"p50_intvty":1017.655},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"1193925.0238225148","tput_per_gpu":2041.6117907365,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":162.2807013289},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"5625382.4556179475","tput_per_gpu":10688.2266656741,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":42.8860162199},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":8464149.421807263,"tput_per_gpu":16081.8839014338,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":32.145951166},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1043891.4142767155","tput_per_gpu":1430.1312375591,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":715.2267858602,"output_tput_per_gpu":714.9044516989,"p50_itl":0.0188203100115,"p90_itl":0.0196715565398,"mean_itl":0.0216253162066,"p50_e2el":20.2547310306691,"p50_tpot":"0.021893404842799998","p50_ttft":"0.10790327796709999","p90_e2el":22.3922826619819,"p90_tpot":0.0224261134717,"p90_ttft":0.2451570102014,"mean_e2el":20.131372951236,"mean_tpot":"0.021610286442100002","mean_ttft":0.2184778272596,"p50_intvty":45.6758556826,"p90_intvty":44.5908739945,"mean_intvty":46.2742593755},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":256,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":2258308.934111168,"tput_per_gpu":3093.8832397323,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":1546.1937014067,"output_tput_per_gpu":1547.6895383255,"p50_itl":"0.032933258451500004","p90_itl":0.0635900789872,"mean_itl":0.0399897241566,"p50_e2el":37.4095293073915,"p50_tpot":0.0405754109644,"p50_ttft":"0.18450535694140002","p90_e2el":41.7439469584264,"p90_tpot":0.0416897596001,"p90_ttft":"0.30147306909780003","mean_e2el":37.2799390947199,"mean_tpot":"0.039947576210199995","mean_ttft":"0.45330854108979995","p50_intvty":24.6454681846,"p90_intvty":23.9867058384,"mean_intvty":25.0328078664},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":8192,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":7598495.802103726,"tput_per_gpu":16108.8111004599,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":24198.1134454066,"output_tput_per_gpu":12064.1599279866,"p50_itl":3.8640122381039,"mean_itl":3.6049946607212,"p50_e2el":36.9142881820444,"p50_tpot":0.0390980583551,"p50_ttft":0.8533751624636,"mean_e2el":37.8563565274574,"mean_tpot":0.0380621323979,"mean_ttft":2.8190385934019,"p50_intvty":25.5767176701,"mean_intvty":26.2728317359},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":6049376.303948916,"tput_per_gpu":12824.6777643717,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":19270.3332550883,"output_tput_per_gpu":9601.8500190135,"p50_itl":2.4604460659903,"mean_itl":2.3311795265023,"p50_e2el":24.1655545254471,"p50_tpot":0.0248542673525,"p50_ttft":"1.1069707325659999","mean_e2el":24.7341130698078,"mean_tpot":0.0246105488259,"mean_ttft":2.0830807888024,"p50_intvty":40.2345394381,"mean_intvty":40.6329825097},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"19510.199518867925","tput_per_gpu":41.36162298,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":186.0431289575,"output_tput_per_gpu":23.2764347328,"p50_itl":0.5228369104443,"mean_itl":"0.49369887496630005","p50_e2el":9.768830527551499,"p50_tpot":0.0052139920516,"p50_ttft":"4.9459781049517995","mean_e2el":9.4899390953127,"mean_tpot":0.0052288339061,"mean_ttft":4.651690278641801,"p50_intvty":191.7916234047,"mean_intvty":191.2472298729},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":1229,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1904191.9508375472","tput_per_gpu":4036.8869357756,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":18187.702219706,"output_tput_per_gpu":2268.0350252843,"p50_itl":1.3016033419408,"mean_itl":1.2435613367717,"p50_e2el":"14.462999968091001","p50_tpot":0.0131334299105,"p50_ttft":2.0422105049947,"mean_e2el":14.858669718022,"mean_tpot":0.0131275487183,"mean_ttft":2.7679955028398,"p50_intvty":76.1415720658,"mean_intvty":76.1756837821},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":4,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":952613.0754385965,"tput_per_gpu":1628.968359,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":12.80197,"input_tput_per_gpu":1447.297,"kv_cache_precision":"FP8","total_input_tokens":74113,"output_tput_per_gpu":181.6713,"total_output_tokens":9303,"tput_requests_per_sec":0.78113,"p50_e2el":"1.3571769999999999","p50_tpot":0.001261,"p50_ttft":0.067069,"p50_intvty":792.9572},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"193282.19080923978","tput_per_gpu":330.5125462838,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":204.8590147099},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"3568184.4761803513","tput_per_gpu":6101.5954542684,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":79.9594965759},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":440378.9315434737,"tput_per_gpu":836.7199699326,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":417.7080579584,"output_tput_per_gpu":419.0119119742,"p50_itl":4.1819910250488,"p90_itl":4.526558781974001,"mean_itl":"3.7344296226858003","p50_e2el":17.4931713009719,"p50_tpot":0.0182016729535,"p50_ttft":0.1842511214782,"p90_e2el":"20.772122321336102","p90_tpot":"0.021404050334299998","p90_ttft":1.8216523942305,"mean_e2el":17.1439950822452,"mean_tpot":0.0178406720121,"mean_ttft":0.6891647761578,"p50_intvty":54.9400048313,"p90_intvty":46.7201293392,"mean_intvty":56.0517002568},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":333,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1067671.0017690095","tput_per_gpu":2263.4625237503,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":19225.5289193477,"output_tput_per_gpu":1203.3333740254,"p50_itl":"1.4978534699766999","mean_itl":1.3238403965757,"p50_e2el":"7.5344988584984005","p50_tpot":0.0063553865245,"p50_ttft":1.6955248714949,"mean_e2el":7.4894653753443,"mean_tpot":"0.0060738859309000005","mean_ttft":1.894749008377,"p50_intvty":157.3468421059,"mean_intvty":164.6392460079},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-22","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"4629883.8085933495","tput_per_gpu":9815.3536742179,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":24580.8822378627,"output_tput_per_gpu":6123.9715333067,"p50_itl":1.9056557889562,"mean_itl":1.7998397884131,"p50_e2el":18.7081139091169,"p50_tpot":0.0192203932701,"p50_ttft":0.9312563450076,"mean_e2el":19.3683190648679,"mean_tpot":0.0190014471277,"mean_ttft":1.8799108034377,"p50_intvty":52.0280717438,"mean_intvty":52.6275705886},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":12,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":36030.7725154717,"tput_per_gpu":76.3852377328,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":345.849717964,"output_tput_per_gpu":42.7021777039,"p50_itl":0.5722004710115,"mean_itl":0.5393864051279,"p50_e2el":7.3411171695043,"p50_tpot":0.0057275536417,"p50_ttft":2.2145244915009,"mean_e2el":7.7798368849673,"mean_tpot":0.005727748941199999,"mean_ttft":"2.5343042540329996","p50_intvty":174.5946109911,"mean_intvty":174.588657825},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-03","num_gpus":12,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2476243.5455916035","tput_per_gpu":5249.6363166542,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13998.6749993308,"output_tput_per_gpu":875.1169753159,"p50_itl":"1.0574833799619001","mean_itl":1.945211358993,"p50_e2el":31.7860468844883,"p50_tpot":0.0095659774569,"p50_ttft":21.9614251834573,"mean_e2el":"31.566958681454402","mean_tpot":0.0108575735101,"mean_ttft":21.570396048759,"p50_intvty":104.5371478766,"mean_intvty":92.1016099104},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":1024,"date":"2026-06-03","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4597214.467291038,"tput_per_gpu":9746.094670657,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":12994.6555126256,"output_tput_per_gpu":3248.9729867197,"p50_itl":1.6842793959659,"mean_itl":3.0710790361309,"p50_e2el":32.0160788504872,"p50_tpot":0.0146483649506,"p50_ttft":16.9976140515646,"mean_e2el":32.6632504613388,"mean_tpot":"0.016642974846800003","mean_ttft":17.3447443291106,"p50_intvty":68.2670047731,"mean_intvty":60.0854119653},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":8192,"date":"2026-06-03","num_gpus":40,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5770501.649172642,"tput_per_gpu":12233.463496246,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":32,"input_tput_per_gpu":13593.0258189676,"output_tput_per_gpu":6795.2142053593,"p50_itl":2.2072680727579,"mean_itl":4.3511903014226,"p50_e2el":132.3086133077741,"p50_tpot":0.037082161998,"p50_ttft":97.3444898827001,"mean_e2el":"131.33281942698937","mean_tpot":0.0374260289568,"mean_ttft":96.8810617861887,"p50_intvty":26.9671439344,"mean_intvty":26.7193722624},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-03-23","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1697750.0129848174","tput_per_gpu":2325.9175177892,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":2756.6677014945,"output_tput_per_gpu":1033.6669666731,"p50_itl":7.2004517865134,"mean_itl":6.5006228946545,"p50_e2el":54.2104033875512,"p50_tpot":0.0400201778799,"p50_ttft":"17.608300240593998","mean_e2el":54.897621553019,"mean_tpot":0.0391853842074,"mean_ttft":18.8090075082845,"p50_intvty":24.9873951836,"mean_intvty":25.5197191562},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":8538115.107889708,"tput_per_gpu":14600.1768344914,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":35.5503075479},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":1229,"date":"2026-02-03","num_gpus":38,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":2816161.067549057,"tput_per_gpu":5970.261463204,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":18902.8626833591,"output_tput_per_gpu":3545.3987344249,"p50_itl":"2.0182318245352002","mean_itl":1.784913617429,"p50_e2el":9.431017198512599,"p50_tpot":0.0085572260866,"p50_ttft":1.557189260493,"mean_e2el":9.413059297616101,"mean_tpot":0.0082014356228,"mean_ttft":1.8647192877487,"p50_intvty":116.8602991057,"mean_intvty":121.9298725235},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":2253,"date":"2026-02-03","num_gpus":22,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":5659108.919853963,"tput_per_gpu":11997.3109100904,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":21986.6229644838,"output_tput_per_gpu":8251.3188896928,"p50_itl":2.5215269179898,"mean_itl":2.2966773211464,"p50_e2el":14.483616410027,"p50_tpot":0.0142063192965,"p50_ttft":1.4014003720076,"mean_e2el":"14.939291313216799","mean_tpot":0.0140232911596,"mean_ttft":2.0196914795335,"p50_intvty":70.3912096533,"mean_intvty":71.3099363492},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"450117.60084405664","tput_per_gpu":954.2493137894,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":5932.8540847809,"output_tput_per_gpu":124.4818519575,"p50_itl":0.020272064954,"mean_itl":0.0201495911956,"p50_e2el":8.8828470648732,"p50_tpot":0.0059083762119,"p50_ttft":2.6911695743911,"mean_e2el":9.3414055819609,"mean_tpot":0.0065568274414,"mean_ttft":3.2744267459435,"p50_intvty":169.2512399587,"mean_intvty":152.5127828874},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":3072,"date":"2026-07-30","num_gpus":40,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5108581.962134105,"tput_per_gpu":9706.3057280548,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":32,"input_tput_per_gpu":10784.9105271054,"output_tput_per_gpu":5391.8865318525,"p50_itl":"1.8122211690060999","mean_itl":"2.9704723552864003","p50_e2el":60.9360859270091,"p50_tpot":"0.028795779821099998","p50_ttft":34.4934052500175,"mean_e2el":60.9791404501453,"mean_tpot":0.0286640683375,"mean_ttft":34.5929915712194,"p50_intvty":34.7273109536,"mean_intvty":34.886883056},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":2198,"date":"2026-01-28","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8351617.862749789,"tput_per_gpu":15868.0739392246,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":20,"input_tput_per_gpu":19744.7876281265,"output_tput_per_gpu":6176.28971697,"p50_itl":7.1040376119781,"mean_itl":6.2812625352905,"p50_e2el":38.5232207069639,"p50_tpot":"0.038480326340399995","p50_ttft":2.7733500495087,"mean_e2el":"39.720939040408304","mean_tpot":"0.037474421129200006","mean_ttft":5.1931420447888,"p50_intvty":25.9873055949,"mean_intvty":26.6848685014},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-07-30","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"3818451.4096879475","tput_per_gpu":7255.0576784071,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":12897.9959660533,"output_tput_per_gpu":1612.119390761,"p50_itl":1.5717418513959,"mean_itl":2.2194849229306,"p50_e2el":33.9472275020089,"p50_tpot":0.0155616967095,"p50_ttft":18.8754059229977,"mean_e2el":"33.240407566606805","mean_tpot":0.0161409227835,"mean_ttft":18.3819964881989,"p50_intvty":64.2603450426,"mean_intvty":61.9543264913},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-07-30","num_gpus":20,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1620396.8729692104","tput_per_gpu":3078.7540586415,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13683.0042026165,"output_tput_per_gpu":427.6915226477,"p50_itl":1.1332159534977,"mean_itl":"1.6030485484349002","p50_e2el":"31.725231921998798","p50_tpot":0.0110943205921,"p50_ttft":20.8943321859988,"mean_e2el":31.6998155883395,"mean_tpot":0.0115753844917,"mean_ttft":21.044552342593,"p50_intvty":90.1362090359,"mean_intvty":86.3902188923},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":2048,"date":"2026-07-30","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4710759.624564263,"tput_per_gpu":8950.4432866721,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":10606.8549536394,"output_tput_per_gpu":3981.2082857699,"p50_itl":2.1522925019963,"mean_itl":2.8342850979808,"p50_e2el":54.2546787114698,"p50_tpot":0.0214378671703,"p50_ttft":33.6257317820564,"mean_e2el":"54.306293817405994","mean_tpot":0.0220398703376,"mean_ttft":34.0011599308729,"p50_intvty":46.6464313849,"mean_intvty":45.3723177443},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-30","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":561215.5857516316,"tput_per_gpu":1066.3096129281,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":6629.566562201,"output_tput_per_gpu":139.1001213826,"p50_itl":0.0173780650366,"mean_itl":"0.017520339783099997","p50_e2el":8.0680750375031,"p50_tpot":0.006482640527,"p50_ttft":"1.9214555880170998","mean_e2el":8.3803951027243,"mean_tpot":0.0067493535906,"mean_ttft":2.1417854457544,"p50_intvty":154.2581292052,"mean_intvty":148.1623368198},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"1710434.3416058396","dataset_name":"random","tput_per_gpu":2343.295048,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":34.95931939,"input_tput_per_gpu":1171.647524,"total_input_tokens":327680,"output_tput_per_gpu":1171.647524,"total_output_tokens":327680,"tput_requests_per_sec":9.15349628,"p50_e2el":6.970084015,"p50_tpot":0.006520527999999999,"p50_ttft":"0.29958349399999995","p50_intvty":153.361805},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-01-28","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"180078.20733578946","tput_per_gpu":342.148593938,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":5162.7298436567,"output_tput_per_gpu":40.8622658305,"p50_itl":0.7757313795155,"mean_itl":0.6886392038410001,"p50_e2el":5.6710503725335,"p50_tpot":0.003138802231,"p50_ttft":"2.9765972014283997","mean_e2el":5.4718045315385,"mean_tpot":0.0029003202073,"mean_ttft":2.7861119946494,"p50_intvty":318.5928664545,"mean_intvty":344.789515819},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":60,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"252643.33433578946","tput_per_gpu":480.022335238,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":5032.2650897642,"output_tput_per_gpu":252.4101975117,"p50_itl":1.3597503451165,"mean_itl":1.2089306918466,"p50_e2el":5.4809079284314,"p50_tpot":0.0057695114784,"p50_ttft":0.1678955900716,"mean_e2el":5.2888067352357,"mean_tpot":"0.0054779270897999995","mean_ttft":0.2374915942849,"p50_intvty":173.3248999915,"mean_intvty":182.5508050047},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":32,"date":"2026-01-28","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"524194.49295321055","tput_per_gpu":995.9695366111,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":15038.3001021433,"output_tput_per_gpu":118.3238762653,"p50_itl":"1.0309762844116999","mean_itl":0.8967610940178999,"p50_e2el":7.4758541445481,"p50_tpot":0.0042205007609,"p50_ttft":3.8992205590475,"mean_e2el":7.3991770729554,"mean_tpot":0.0038995233628,"mean_ttft":3.7953188362582,"p50_intvty":236.9387086155,"mean_intvty":256.4415973326},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-07-30","num_gpus":12,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2667436.0012146314","tput_per_gpu":5068.1284023078,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13514.6661748183,"output_tput_per_gpu":844.8595160525,"p50_itl":1.2634256619494,"mean_itl":1.7943623617921,"p50_e2el":31.2217776040779,"p50_tpot":"0.012631354326900001","p50_ttft":18.84825535852,"mean_e2el":31.9554837974658,"mean_tpot":0.0131355933148,"mean_ttft":"19.858257328400303","p50_intvty":79.1680744693,"mean_intvty":76.1290317107},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-03-23","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1690299.8447402923","tput_per_gpu":2315.7107872942,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":3473.4638911102,"output_tput_per_gpu":1736.8342353862,"p50_itl":5.612508881022199,"mean_itl":5.2264811988639,"p50_e2el":31.6262722790707,"p50_tpot":0.0319316036048,"p50_ttft":1.831923855003,"mean_e2el":32.5972325270066,"mean_tpot":0.0319947703509,"mean_ttft":3.139069723424,"p50_intvty":31.3169364238,"mean_intvty":31.255107914},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":201444.7283941606,"dataset_name":"random","tput_per_gpu":275.9792779,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":18.55211753,"input_tput_per_gpu":137.9896389,"total_input_tokens":10240,"output_tput_per_gpu":137.9896389,"total_output_tokens":10240,"tput_requests_per_sec":0.539022027,"p50_e2el":3.709422564,"p50_tpot":0.003589207,"p50_ttft":0.037663763,"p50_intvty":278.6130722},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":256,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":22605032.59649123,"tput_per_gpu":38654.60574,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":122.514,"input_tput_per_gpu":19375.64,"kv_cache_precision":"FP8","total_input_tokens":2373787,"output_tput_per_gpu":19278.96,"total_output_tokens":2361942,"tput_requests_per_sec":20.89558,"p50_e2el":12.93751,"p50_tpot":0.012546,"p50_ttft":"0.10297400000000001","p50_intvty":79.70679},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"254723.40192982458","tput_per_gpu":435.5770173,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":10.54865,"input_tput_per_gpu":217.3501,"kv_cache_precision":"FP8","total_input_tokens":9171,"output_tput_per_gpu":218.227,"total_output_tokens":9208,"tput_requests_per_sec":0.947989,"p50_e2el":1.123263,"p50_tpot":0.00106,"p50_ttft":0.038532,"p50_intvty":943.0912},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":1024,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"11690090.497344999","tput_per_gpu":22211.1719449555,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":22.1701251209},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"37713538.169590645","tput_per_gpu":64490.15027,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":329.4641,"input_tput_per_gpu":57325.18,"kv_cache_precision":"FP8","total_input_tokens":18886588,"output_tput_per_gpu":7164.975,"total_output_tokens":2360602,"tput_requests_per_sec":7.770194,"p50_e2el":35.28142,"p50_tpot":0.034229,"p50_ttft":0.26506,"p50_intvty":29.21491},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"393026.63765713456","tput_per_gpu":672.0755503937,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":130.263866957},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":393512.9575219883,"tput_per_gpu":672.9071573626,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":130.1280332183},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2004879.3551458397","tput_per_gpu":2746.6847165498,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":24.7752260273},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1497867.5512747448","tput_per_gpu":2052.0785452464,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":117.249997913},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":95645.73099642337,"tput_per_gpu":131.0346514651,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":124.9045994283},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1534964.1624744528","tput_per_gpu":2102.90090259,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":137.5382215279},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":73059.41003846716,"tput_per_gpu":100.0913917527,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":300.9730757744},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":5595799.83895269,"tput_per_gpu":9568.8177246091,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":40.9644202386},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"4035826.7061335766","tput_per_gpu":5529.082587403,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":43.5565945642},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":7759056.902640074,"tput_per_gpu":10629.9079566169,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":5.30107158},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":283277.5699298541,"tput_per_gpu":388.0902708039,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":90.5563295506},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":32,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":720523.4570332847,"tput_per_gpu":987.1171361356,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":30.4722272525},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":7308953.617269636,"tput_per_gpu":10013.2664556594,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":9.5780493514},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"2190012.6172754746","tput_per_gpu":3000.3172856674,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":90.0229111009},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"420442.13480051095","tput_per_gpu":576.0057246767,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":34.2915503549},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":2,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"115098.83136445256","tput_per_gpu":157.6853989693,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":77.9984779377},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":283932.6677031387,"tput_per_gpu":388.9877547533,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":211.701473006},{"ep":4,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-30","num_gpus":16,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1239114.6898542566","tput_per_gpu":1208.1368226079,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":1074.3052085674,"output_tput_per_gpu":133.8316140405,"p50_itl":0.8999792760005,"p90_itl":3.4120660832035,"mean_itl":1.4805348748227,"p50_e2el":26.6324077765166,"p50_tpot":"0.026606008396299997","p50_ttft":2.5393989189761,"p90_e2el":"31.631239744304903","p90_tpot":"0.029170481281699998","p90_ttft":4.7399697384972,"mean_e2el":26.7934996945991,"mean_tpot":"0.025802194617800003","mean_ttft":3.1697152252371,"p50_intvty":37.5854951673,"p90_intvty":34.2812307532,"mean_intvty":38.7563931988},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":128,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":3691032.445351533,"tput_per_gpu":5056.7144501316,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":4496.6049454932,"output_tput_per_gpu":560.1095046384,"p50_itl":0.034847789444,"p90_itl":0.1736481864937,"mean_itl":0.0546538307951,"p50_e2el":51.6661471868865,"p50_tpot":"0.055758145439400005","p50_ttft":0.4683341686614,"p90_e2el":"59.845184174086896","p90_tpot":0.0587158594756,"p90_ttft":1.5199102781713,"mean_e2el":51.7881275206739,"mean_tpot":"0.054571633041399996","mean_ttft":1.5554502009385,"p50_intvty":17.9345993687,"p90_intvty":17.0311736715,"mean_intvty":18.3245386709},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":2048,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"15055701.740882896","tput_per_gpu":28605.8333076775,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":14.7000137363},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":408822.8859940146,"tput_per_gpu":560.0873538118,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":69.4047299976},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":512,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":3157859.02279,"tput_per_gpu":4326.2668612223,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":16.0071821281},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":128,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":67099.05660377358,"tput_per_gpu":142.25,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":137.1742},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":1024,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"124877.35849056605","tput_per_gpu":264.74,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":26.4971},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":512,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"113825.47169811321","tput_per_gpu":241.31,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":44.0723},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":128,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":69405.6603773585,"tput_per_gpu":147.14,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":132.626},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":434948.2664636316,"tput_per_gpu":826.4017062809,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":417.993657404,"output_tput_per_gpu":408.4080488768,"p50_itl":0.08878568396900001,"p90_itl":"0.13666583118029998","mean_itl":0.0940803728348,"p50_e2el":"4.0113132025580995","p50_tpot":"0.0043021694368999995","p50_ttft":"0.057397139025900004","p90_e2el":5.602825083374,"p90_tpot":0.005904196991,"p90_ttft":0.1268760807579,"mean_e2el":4.3038492639433,"mean_tpot":0.004563334859,"mean_ttft":0.1149207528622,"p50_intvty":232.4408684174,"p90_intvty":169.3710425873,"mean_intvty":219.1379837105},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"162641.82249934308","tput_per_gpu":222.8192968241,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":111.9517694495,"output_tput_per_gpu":110.8675273746,"p50_itl":0.008314144797599999,"p90_itl":0.008389848284399998,"mean_itl":0.008590959427799999,"p50_e2el":8.0527864224277,"p50_tpot":0.0085476263449,"p50_ttft":0.1055877376348,"p90_e2el":8.816021524742201,"p90_tpot":0.008634618206500001,"p90_ttft":0.1430304615758,"mean_e2el":8.0036116940202,"mean_tpot":0.008588038570200001,"mean_ttft":0.1280645821709,"p50_intvty":116.9915435754,"p90_intvty":115.8128797452,"mean_intvty":116.4410233864},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":2836607.196792774,"tput_per_gpu":3886.1518596061,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":3455.0172266045,"output_tput_per_gpu":431.1346330016,"p50_itl":0.0256819576025,"p90_itl":0.0267778942361,"mean_itl":"0.035495603323799994","p50_e2el":33.8644896750338,"p50_tpot":0.0362266082779,"p50_ttft":0.3112456756644,"p90_e2el":38.22387230657041,"p90_tpot":0.0376516179793,"p90_ttft":1.1354788580909,"mean_e2el":33.6329334210182,"mean_tpot":0.0354544591299,"mean_ttft":0.9499685740257999,"p50_intvty":27.6040194635,"p90_intvty":26.5592836024,"mean_intvty":28.2051968791},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":8192,"osl":1024,"conc":308,"date":"2026-02-03","num_gpus":48,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"3291308.7043641983","tput_per_gpu":6977.5744532521,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":18606.0611667026,"output_tput_per_gpu":1163.3310965269,"p50_itl":1.6312592354952,"mean_itl":1.4225727403372,"p50_e2el":7.300424157991101,"p50_tpot":0.0066564639831,"p50_ttft":"1.1916593965142999","mean_e2el":7.1409414125765,"mean_tpot":0.0061745979212,"mean_ttft":"1.4561931011256999","p50_intvty":150.2299122392,"mean_intvty":161.9538652988},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":1127,"date":"2026-02-03","num_gpus":42,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":6863618.555416368,"tput_per_gpu":14550.8713374827,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":26,"input_tput_per_gpu":20893.1541032793,"output_tput_per_gpu":4244.6618430632,"p50_itl":3.6229923940264,"mean_itl":3.1212061315337,"p50_e2el":14.8042022324807,"p50_tpot":0.0145575827842,"p50_ttft":"1.3658870729560002","mean_e2el":14.5532579171772,"mean_tpot":"0.013527406277300001","mean_ttft":2.101390356178,"p50_intvty":68.69272288,"mean_intvty":73.9240013572},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":1024,"date":"2026-07-30","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4202595.190386211,"tput_per_gpu":7984.9308617338,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":10646.4619262075,"output_tput_per_gpu":2661.8687327864,"p50_itl":1.8693808289972,"mean_itl":2.6216734800207,"p50_e2el":"41.510706752000296","p50_tpot":0.0182257741754,"p50_ttft":"23.981834394500797","mean_e2el":40.9722175821437,"mean_tpot":0.0189413495581,"mean_ttft":23.5442335834333,"p50_intvty":54.8673538021,"mean_intvty":52.7945486109},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-30","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":841501.0886263158,"tput_per_gpu":1598.85206839,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":9950.4631206267,"output_tput_per_gpu":206.9168930172,"p50_itl":0.0213214129908,"mean_itl":0.0217153709736,"p50_e2el":10.8408225479652,"p50_tpot":0.0080565869994,"p50_ttft":3.0273493489367,"mean_e2el":11.3294349768914,"mean_tpot":0.008402739584600001,"mean_ttft":3.5943520511837,"p50_intvty":124.1220382869,"mean_intvty":119.0088053942},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-30","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":4,"tput_per_mw":"107635.60939178948","tput_per_gpu":204.5076578444,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":363.3285318134,"output_tput_per_gpu":45.6867838755,"p50_itl":0.012931571051,"mean_itl":"0.012939900913900002","p50_e2el":5.1726104334812,"p50_tpot":0.0052060036637,"p50_ttft":"0.32396186247930003","mean_e2el":5.0897878156975,"mean_tpot":0.0051148094463,"mean_ttft":0.3537844012026,"p50_intvty":192.0859193736,"mean_intvty":195.5107048462},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":200,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":501823.7716383158,"tput_per_gpu":953.4651661128,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":10018.4850214547,"output_tput_per_gpu":500.2141733457,"p50_itl":1.8583798034815,"mean_itl":1.6333709418746,"p50_e2el":8.869902378530199,"p50_tpot":0.007854211999,"p50_ttft":1.7260119415587,"mean_e2el":8.7824914097455,"mean_tpot":0.007515602613,"mean_ttft":1.8782327033884,"p50_intvty":127.3202200464,"mean_intvty":133.0565293958},{"ep":1,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-30","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":748306.920507795,"tput_per_gpu":729.5992474951,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":648.459423161,"output_tput_per_gpu":81.139824334,"p50_itl":"0.38404159800849996","p90_itl":"1.4475693690473999","mean_itl":0.6108011500878,"p50_e2el":"10.824106211017401","p50_tpot":0.0109199703179,"p50_ttft":0.9744464899995,"p90_e2el":12.6907107649255,"p90_tpot":"0.012251778855099999","p90_ttft":1.8001281432982,"mean_e2el":11.136933345873,"mean_tpot":0.010830155758,"mean_ttft":1.1961447821202,"p50_intvty":91.5753404896,"p90_intvty":81.6208006879,"mean_intvty":92.3347754498},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":740194.2536746842,"tput_per_gpu":1406.3690819819,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":1248.8496655771,"output_tput_per_gpu":157.5194164047,"p50_itl":"0.052837271010499995","p90_itl":0.1085667668842,"mean_itl":"0.062427672528700004","p50_e2el":"2.4746563304216003","p50_tpot":0.00239724057,"p50_ttft":0.1623500814894,"p90_e2el":4.8967628324404,"p90_tpot":"0.0049074222966000005","p90_ttft":0.1765155840199,"mean_e2el":2.9428587093018,"mean_tpot":0.0029571002064,"mean_ttft":"0.16794876190830002","p50_intvty":417.1462858305,"p90_intvty":203.772966653,"mean_intvty":338.1691286092},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":372167.1104559124,"tput_per_gpu":509.8689413246,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":452.6540168397,"output_tput_per_gpu":57.214924485,"p50_itl":0.0083000967279,"p90_itl":0.0083814170212,"mean_itl":0.0084523734047,"p50_e2el":8.21787916217,"p50_tpot":0.008453278903399999,"p50_ttft":0.2390692718327,"p90_e2el":8.7849306316115,"p90_tpot":0.008519682939100002,"p90_ttft":"0.33419705508279995","mean_e2el":8.1502754493617,"mean_tpot":0.008448617346100001,"mean_ttft":0.2553363617044,"p50_intvty":118.2972916694,"p90_intvty":117.375259989,"mean_intvty":118.3625626575},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":276106.6887718249,"tput_per_gpu":378.2661636174,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":188.4319882325,"output_tput_per_gpu":189.8341753849,"p50_itl":0.0097422655672,"p90_itl":"0.010233982931799999","mean_itl":0.0101999634398,"p50_e2el":9.6386724147014,"p50_tpot":0.0102459912124,"p50_ttft":0.1050950782374,"p90_e2el":10.454299660679,"p90_tpot":"0.010332896073200001","p90_ttft":0.1618029829115,"mean_e2el":9.5899122890551,"mean_tpot":0.010196474018,"mean_ttft":0.1235814067884,"p50_intvty":97.5991467556,"p90_intvty":96.7782887701,"mean_intvty":98.0731180445},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1824337.8589446715","tput_per_gpu":2499.3428667542,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":1251.9927872426,"output_tput_per_gpu":1247.3500795116,"p50_itl":0.0223638750613,"p90_itl":0.0235716090538,"mean_itl":"0.023831605747000002","p50_e2el":22.1437897494761,"p50_tpot":0.0241954967864,"p50_ttft":0.0992626720108,"p90_e2el":24.6073515419383,"p90_tpot":0.0250509520296,"p90_ttft":0.4236580020282,"mean_e2el":22.0791437161743,"mean_tpot":0.0238234929857,"mean_ttft":0.1969829663728,"p50_intvty":41.3300048695,"p90_intvty":39.9186425657,"mean_intvty":41.9753728221},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":10,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":67461.27730263158,"tput_per_gpu":128.176426875,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":1351.5168321981,"output_tput_per_gpu":67.0094066089,"p50_itl":0.8281047309283,"mean_itl":0.7350380142327,"p50_e2el":3.368082718458,"p50_tpot":0.0034741356025,"p50_ttft":"0.11814102949569999","mean_e2el":3.2511560030095,"mean_tpot":0.0033617640816,"mean_ttft":"0.17134704606610002","p50_intvty":287.8413839903,"mean_intvty":297.4628723871},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1026654.0812368614","tput_per_gpu":1406.5160912945,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":90.7973850019},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":256,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":99683.96226415095,"tput_per_gpu":211.33,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":70.8717},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":512,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":95424.5283018868,"tput_per_gpu":202.3,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":85.6898},{"ep":1,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-29","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":3560183.860142211,"tput_per_gpu":6764.3493342702,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":6015.4525579199,"output_tput_per_gpu":748.8967763503,"p50_itl":0.1622783179628,"p90_itl":"0.35936415998730004","mean_itl":0.2049194713453,"p50_e2el":8.7741361965018,"p50_tpot":0.009193325569,"p50_ttft":0.2266005139973,"p90_e2el":"13.305372092902001","p90_tpot":0.0140033295732,"p90_ttft":0.5437862758001001,"mean_e2el":9.521125786379299,"mean_tpot":0.0099602933619,"mean_ttft":0.4009286496927,"p50_intvty":108.7745661232,"p90_intvty":71.4115878491,"mean_intvty":100.3986492832},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":901710.0204862045,"tput_per_gpu":1235.3427280661,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":624.7349132528,"output_tput_per_gpu":610.6078148133,"p50_itl":0.4808022361249,"p90_itl":0.7771133292465999,"mean_itl":0.5098206584038,"p50_e2el":22.2675229087472,"p50_tpot":0.0234480924394,"p50_ttft":"0.28398462943729996","p90_e2el":30.7647016445175,"p90_tpot":0.0327709800428,"p90_ttft":0.5465504579246,"mean_e2el":23.3309801179945,"mean_tpot":"0.024801244655499998","mean_ttft":0.5666913128633,"p50_intvty":42.6473924302,"p90_intvty":30.5148029963,"mean_intvty":40.3205570483},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":89417.88118708029,"tput_per_gpu":122.5024972263,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":61.068409065,"output_tput_per_gpu":61.4340881612,"p50_itl":0.0807208530605,"p90_itl":"0.11146183647220001","mean_itl":0.0803455905598,"p50_e2el":4.0695190858096,"p50_tpot":0.004074941704699999,"p50_ttft":0.1198650505394,"p90_e2el":4.6432319236919,"p90_tpot":0.0048417562344,"p90_ttft":"0.12538996990770002","mean_e2el":3.7596739953384,"mean_tpot":0.0039413208264,"mean_ttft":0.1200189558789,"p50_intvty":245.4022836333,"p90_intvty":206.5366267103,"mean_intvty":253.7220500522},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":615,"date":"2026-07-22","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":6545431.261280468,"tput_per_gpu":11192.6874567896,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":19897.6019397654,"output_tput_per_gpu":2487.7729738139,"p50_itl":2.4837234345014,"mean_itl":2.3470633870085,"p50_e2el":27.5861520540002,"p50_tpot":"0.024874296682899998","p50_ttft":4.6577012415,"mean_e2el":"27.874815680981598","mean_tpot":0.0247795628747,"mean_ttft":5.0525821139407,"p50_intvty":40.2021417026,"mean_intvty":40.3558369878},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-22","num_gpus":36,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":96991.532305731,"tput_per_gpu":165.8555202428,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":1324.996872809,"output_tput_per_gpu":20.962851172,"p50_itl":0.5403338299948,"mean_itl":0.5118868017539,"p50_e2el":5.5282595185054,"p50_tpot":0.005428261617000001,"p50_ttft":"0.40693402900799996","mean_e2el":5.468217146775,"mean_tpot":0.0054238046485,"mean_ttft":0.4517266986875,"p50_intvty":184.2210399129,"mean_intvty":184.3724220911},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"488482.31295620446","tput_per_gpu":669.22076875,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":27.9659859892},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":256,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":3199072.32383731,"tput_per_gpu":5470.4136737618,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":100.2631443962},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"124196.79582372264","tput_per_gpu":170.1496102785,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":143.5998097025},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":56,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":50683.96226415095,"tput_per_gpu":107.45,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":173.3102},{"ep":4,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-30","num_gpus":16,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1773642.2274213333","tput_per_gpu":1729.3011717358,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":1537.5054484159,"output_tput_per_gpu":191.7957233199,"p50_itl":2.1665497754584,"p90_itl":9.992794622929098,"mean_itl":4.078385031871,"p50_e2el":76.3762338670203,"p50_tpot":0.0769460248014,"p50_ttft":5.8141274949885,"p90_e2el":88.4930852685415,"p90_tpot":0.0850424379888,"p90_ttft":14.8179272119829,"mean_e2el":76.016389290229,"mean_tpot":0.07354872872679999,"mean_ttft":8.2642180621348,"p50_intvty":12.9961229652,"p90_intvty":11.7588350434,"mean_intvty":13.5964280731},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1749201.7552742632","tput_per_gpu":3323.4833350211,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":2950.8582473639,"output_tput_per_gpu":372.6250876572,"p50_itl":0.0825951349689,"p90_itl":0.1900543764001,"mean_itl":0.1019775946732,"p50_e2el":4.4182118964964,"p50_tpot":0.0045180771853,"p50_ttft":0.1631984959822,"p90_e2el":"6.5917346208356005","p90_tpot":0.007017312805699999,"p90_ttft":0.2747640188085,"mean_e2el":4.8114571412661,"mean_tpot":0.0049677294164,"mean_ttft":0.2160920196766,"p50_intvty":221.3330934784,"p90_intvty":142.5046919947,"mean_intvty":201.2992085896},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":24,"date":"2026-07-22","num_gpus":36,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":335031.6794557895,"tput_per_gpu":572.9041718694,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":4581.2755753804,"output_tput_per_gpu":71.8577464305,"p50_itl":0.6656299320002,"mean_itl":0.6339778713782,"p50_e2el":9.3489414944997,"p50_tpot":0.0067010362976,"p50_ttft":3.0959635114996,"mean_e2el":9.4639190604541,"mean_tpot":0.006693204760100001,"mean_ttft":"3.2720686972707997","p50_intvty":149.2306496474,"mean_intvty":149.4052603858},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":1127,"date":"2026-07-22","num_gpus":20,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":8402438.759990761,"tput_per_gpu":14368.1702795842,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":21284.6270049268,"output_tput_per_gpu":3993.4851915705,"p50_itl":"3.0853700519947003","mean_itl":2.9101942365412,"p50_e2el":31.4732554590009,"p50_tpot":0.0308935880857,"p50_ttft":2.9279764940002,"mean_e2el":"31.856907381081097","mean_tpot":0.0307242260378,"mean_ttft":3.5464244174697,"p50_intvty":32.3691763231,"mean_intvty":32.5476058785},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":2151,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":9708670.694792924,"tput_per_gpu":16601.8268880959,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":22137.3971802755,"output_tput_per_gpu":5530.6863037368,"p50_itl":4.466269871001701,"mean_itl":4.1932871731965,"p50_e2el":43.3713285795002,"p50_tpot":0.0447187170177,"p50_ttft":2.1433708479999,"mean_e2el":44.0053452532375,"mean_tpot":0.044267797518,"mean_ttft":3.2614489999496,"p50_intvty":22.3620011192,"mean_intvty":22.5897843595},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":5,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"173595.45872409357","tput_per_gpu":296.8482344182,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":1582.9033500857,"output_tput_per_gpu":39.6372112847,"p50_itl":0.5800941140005,"mean_itl":0.5488846017574,"p50_e2el":"5.7129023834999995","p50_tpot":0.0058039669036,"p50_ttft":"0.33613333399990003","mean_e2el":5.711121141259899,"mean_tpot":0.0058307766171,"mean_ttft":"0.37047417708990005","p50_intvty":172.2959514774,"mean_intvty":171.5037405251},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"129082.45485839417","tput_per_gpu":176.842963156,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":261.1711228229},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":663067.9905259855,"tput_per_gpu":908.4031470206,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":123.7343047309},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":456507.7837735766,"tput_per_gpu":625.4156637698,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":163.3362192954},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":128,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":9673880.998173157,"tput_per_gpu":18380.373896529,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":16345.1785629903,"output_tput_per_gpu":2035.1953335388,"p50_itl":1.5145233978983,"p90_itl":5.0444335522596,"mean_itl":2.2670691192902,"p50_e2el":28.7253149839817,"p50_tpot":"0.028999493489199998","p50_ttft":2.1463043739786,"p90_e2el":33.7471807079157,"p90_tpot":0.0322701482748,"p90_ttft":4.2644590886892,"mean_e2el":28.5222007547879,"mean_tpot":"0.027940673487200002","mean_ttft":2.8281604756869,"p50_intvty":34.4833609033,"p90_intvty":30.9883918563,"mean_intvty":35.7901179604},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":711822.1617812632,"tput_per_gpu":1352.4621073844,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":681.6841628475,"output_tput_per_gpu":670.777944537,"p50_itl":0.1146327080205,"p90_itl":0.1674268919975,"mean_itl":0.1164744320691,"p50_e2el":5.1694164657965,"p50_tpot":0.005593879700099999,"p50_ttft":0.06486689136360001,"p90_e2el":6.8018064265605,"p90_tpot":0.0072217399779,"p90_ttft":0.0964768314734,"mean_e2el":5.3343404652289,"mean_tpot":"0.0056555008303000006","mean_ttft":"0.13157326619259999","p50_intvty":178.7668047257,"p90_intvty":138.470784473,"mean_intvty":176.8189997686},{"ep":4,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-13","num_gpus":16,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1041295.9266197812","tput_per_gpu":1426.5754194691,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":721.2882977528,"output_tput_per_gpu":705.2871217162,"p50_itl":0.8377594547346,"p90_itl":1.3278530221432,"mean_itl":0.8842741311703001,"p50_e2el":38.7962189232931,"p50_tpot":0.0413710032003,"p50_ttft":0.4048118656501,"p90_e2el":"54.567698247916994","p90_tpot":0.0571102055949,"p90_ttft":0.7549702767283,"mean_e2el":"40.533405161750906","mean_tpot":0.043007722751000003,"mean_ttft":1.0326035507649,"p50_intvty":24.171519244,"p90_intvty":17.5100052536,"mean_intvty":23.2516379858},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":89237.7745970073,"tput_per_gpu":122.2557511979,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":60.752148704,"output_tput_per_gpu":61.5036024939,"p50_itl":0.007892882335,"p90_itl":0.0079372975742,"mean_itl":0.008070915198,"p50_e2el":7.668909486616,"p50_tpot":0.0079409422279,"p50_ttft":0.07288729201540001,"p90_e2el":8.033171174,"p90_tpot":0.0079572950329,"p90_ttft":0.3552852850407,"mean_e2el":7.5071199861413,"mean_tpot":0.0079222952845,"mean_ttft":"0.13938341671020002","p50_intvty":125.9296405007,"p90_intvty":125.6708461686,"mean_intvty":126.2260448632},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":654998.4584432117,"tput_per_gpu":897.3478880672,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":797.5250381169,"output_tput_per_gpu":99.8228499503,"p50_itl":0.0088844699785,"p90_itl":0.0091796418652,"mean_itl":0.0094369900675,"p50_e2el":8.959557075053501,"p50_tpot":0.009441470707,"p50_ttft":0.2348052817397,"p90_e2el":9.8436868653633,"p90_tpot":0.0095654887753,"p90_ttft":0.7861975970678,"mean_e2el":8.978728340612701,"mean_tpot":0.0094296535617,"mean_ttft":0.3077861146536,"p50_intvty":105.9157022278,"p90_intvty":104.5424884699,"mean_intvty":106.0484347022},{"ep":1,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-29","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"248694.47342947367","tput_per_gpu":472.519499516,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":419.9539229728,"output_tput_per_gpu":52.5655765431,"p50_itl":0.0419262647629,"p90_itl":0.0713054453954,"mean_itl":0.0463631660009,"p50_e2el":2.0984918521717,"p50_tpot":0.0021615466186,"p50_ttft":0.1430950150825,"p90_e2el":3.2481147008017,"p90_tpot":0.0030789573797,"p90_ttft":"0.15343933245170002","mean_e2el":2.2119091743603,"mean_tpot":"0.0022096291083000003","mean_ttft":0.1441120729782,"p50_intvty":462.6317061136,"p90_intvty":324.785268736,"mean_intvty":452.5646391228},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":60,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":914253.2710942106,"tput_per_gpu":1563.3730935711,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":8342.0830303226,"output_tput_per_gpu":207.6311062208,"p50_itl":0.9309318710002,"mean_itl":0.8828757009571,"p50_e2el":12.93190624,"p50_tpot":0.009328516220599999,"p50_ttft":"4.3341866069999995","mean_e2el":"12.930386317324999","mean_tpot":0.0093162293834,"mean_ttft":"4.3731137654042005","p50_intvty":107.1981841864,"mean_intvty":107.3395639847},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":195,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2185277.9894821052","tput_per_gpu":3736.8253620144,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":19928.6765847309,"output_tput_per_gpu":498.4551174711,"p50_itl":1.5439094949998,"mean_itl":"1.4590739188125001","p50_e2el":17.4471718980001,"p50_tpot":0.0154783234537,"p50_ttft":"3.1697739604999002","mean_e2el":17.6719075946895,"mean_tpot":0.0154048663193,"mean_ttft":3.4848457142215,"p50_intvty":64.6064803459,"mean_intvty":64.9145522766},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":115,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1453565.2197523394","tput_per_gpu":2485.5965257765,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13256.5232206084,"output_tput_per_gpu":331.4111868102,"p50_itl":"1.1678482140000999","mean_itl":1.1061295004695,"p50_e2el":15.5621356224999,"p50_tpot":0.0117059980486,"p50_ttft":"4.7689566789999995","mean_e2el":15.6413336568083,"mean_tpot":0.0116785282695,"mean_ttft":4.878693802668701,"p50_intvty":85.4262913634,"mean_intvty":85.6272277567},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":5983439.984087158,"tput_per_gpu":11368.5359697656,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":10097.5936035669,"output_tput_per_gpu":1270.9423661987,"p50_itl":0.6497619119472999,"p90_itl":"1.9614596794824999","mean_itl":0.9068100987223,"p50_e2el":11.5016502371291,"p50_tpot":0.0113443581717,"p50_ttft":"1.0055775489891001","p90_e2el":13.1733580695,"p90_tpot":0.012597778795,"p90_ttft":1.7950015180279,"mean_e2el":11.3727106397862,"mean_tpot":0.0110204340402,"mean_ttft":"1.1795985977121999","p50_intvty":88.1495440171,"p90_intvty":79.3790727934,"mean_intvty":90.740527674},{"ep":1,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-30","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"319614.98865302565","tput_per_gpu":311.6246139367,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":276.8209141221,"output_tput_per_gpu":34.8036998146,"p50_itl":0.265498310473,"p90_itl":0.8216839288827,"mean_itl":0.3705883637281,"p50_e2el":6.6765127645049,"p50_tpot":0.0065118032475,"p50_ttft":0.8304922664829001,"p90_e2el":7.4233098641736,"p90_tpot":0.0066886777493,"p90_ttft":0.8850559312734,"mean_e2el":"6.6821455325931005","mean_tpot":0.0063000585276,"mean_ttft":0.8268495927798,"p50_intvty":153.5672934205,"p90_intvty":149.5063804059,"mean_intvty":158.7286841256},{"ep":1,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-07-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"55563.449513052634","tput_per_gpu":105.5705540748,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":52.6277090462,"output_tput_per_gpu":52.9428450286,"p50_itl":0.0464140969852,"p90_itl":0.0739135743119,"mean_itl":0.0479051239063,"p50_e2el":"1.9648873400001001","p50_tpot":0.002110337162,"p50_ttft":"0.035179897502499995","p90_e2el":"2.9709021829533997","p90_tpot":0.0031074270213,"p90_ttft":0.0359827893262,"mean_e2el":2.1812883270904,"mean_tpot":0.0023286963585,"mean_ttft":0.0351388811076,"p50_intvty":473.8579303873,"p90_intvty":321.8096493111,"mean_intvty":429.42481374},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1236126.3190227367","tput_per_gpu":2348.6400061432,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":2087.9653674489,"output_tput_per_gpu":260.6746386944,"p50_itl":"0.061883019996500005","p90_itl":0.1134991882951,"mean_itl":0.0732896424694,"p50_e2el":3.2377525820048,"p50_tpot":0.0032250357246,"p50_ttft":0.1657148399972,"p90_e2el":4.5271813940984,"p90_tpot":0.0045736610523,"p90_ttft":0.1963182362175,"mean_e2el":3.4756513625507,"mean_tpot":0.0035656045227,"mean_ttft":0.1922755676729,"p50_intvty":310.0740845657,"p90_intvty":218.6432244485,"mean_intvty":280.4573512349},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-13","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"143881.66482277375","tput_per_gpu":197.1178808072,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":99.6864223465,"output_tput_per_gpu":97.4314584607,"p50_itl":0.0897556636482,"p90_itl":0.1449015092105,"mean_itl":0.0994740133336,"p50_e2el":4.4551508771256,"p50_tpot":0.0047523246361,"p50_ttft":0.1550979726017,"p90_e2el":5.4015494091436,"p90_tpot":0.0060419254721,"p90_ttft":0.1613325051963,"mean_e2el":4.5926973733585,"mean_tpot":0.0048624729857,"mean_ttft":0.1611304226331,"p50_intvty":210.4233352235,"p90_intvty":165.5101514607,"mean_intvty":205.6566695474},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-06-13","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":85774.53042583942,"tput_per_gpu":117.5111066834,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":58.32496006,"output_tput_per_gpu":59.1861466234,"p50_itl":0.0785628259182,"p90_itl":0.1182245377451,"mean_itl":0.08249013608900001,"p50_e2el":3.6779948063195,"p50_tpot":0.0038983449457,"p50_ttft":0.1501168757677,"p90_e2el":5.0601628156379,"p90_tpot":0.0050676699151,"p90_ttft":"0.18832175126300002","mean_e2el":3.9437698011287,"mean_tpot":0.0040001504473,"mean_ttft":0.1780952871311,"p50_intvty":256.5191161713,"p90_intvty":197.329347954,"mean_intvty":249.9905973958},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-13","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":50008.64495970803,"tput_per_gpu":68.5118435948,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":34.1536653144,"output_tput_per_gpu":34.3581782804,"p50_itl":0.0695579815656,"p90_itl":0.1041271574795,"mean_itl":0.0720206276371,"p50_e2el":3.413194777444,"p50_tpot":"0.0033985322222000003","p50_ttft":0.1214528419077,"p90_e2el":4.0537216506898,"p90_tpot":0.004277853394900001,"p90_ttft":0.1431099385023,"mean_e2el":3.361127647385,"mean_tpot":0.0034998208496,"mean_ttft":0.1274016674608,"p50_intvty":294.2446722926,"p90_intvty":233.7621016174,"mean_intvty":285.7289109887},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":30,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":562816.2470439181,"tput_per_gpu":962.4157824451,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":5133.1024237569,"output_tput_per_gpu":128.2784541828,"p50_itl":0.7563211830002,"mean_itl":0.7162829572899,"p50_e2el":10.4405883985,"p50_tpot":0.007578686539,"p50_ttft":"3.4798172000005003","mean_e2el":10.5271828419983,"mean_tpot":"0.0075695806186999995","mean_ttft":3.5481994122916,"p50_intvty":131.9489854687,"mean_intvty":132.1077151269},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":308,"date":"2026-07-22","num_gpus":12,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5245682.094343392,"tput_per_gpu":8970.1163813272,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":23922.0971537364,"output_tput_per_gpu":1494.1259951226,"p50_itl":2.050800728,"mean_itl":1.940499205227,"p50_e2el":22.8968203734998,"p50_tpot":0.0205363822259,"p50_ttft":"3.9463532674999002","mean_e2el":23.2203901802279,"mean_tpot":0.0204765829484,"mean_ttft":4.3660464587005,"p50_intvty":48.6940683612,"mean_intvty":48.8362732454},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":4390622.07646621,"tput_per_gpu":8342.1819452858,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":7418.0749621484,"output_tput_per_gpu":924.1069831373,"p50_itl":0.4632717685308,"p90_itl":1.3481292252429,"mean_itl":0.6236730146727,"p50_e2el":7.777872251113901,"p50_tpot":0.007773719703400001,"p50_ttft":0.6845782310702,"p90_e2el":8.9584650473669,"p90_tpot":0.0086202572741,"p90_ttft":1.4753814581782,"mean_e2el":7.7176628048182,"mean_tpot":0.0075539215053,"mean_ttft":0.8026886011387,"p50_intvty":128.6385460435,"p90_intvty":116.005818412,"mean_intvty":132.3815715185},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":1104904.032300421,"tput_per_gpu":2099.3176613708,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":1055.9927875846,"output_tput_per_gpu":1043.3248737862,"p50_itl":0.1391694738995,"p90_itl":0.2229427448008,"mean_itl":0.1488687770613,"p50_e2el":6.475469812518,"p50_tpot":"0.0069246904122000005","p50_ttft":0.0722286410164,"p90_e2el":9.0681647268124,"p90_tpot":0.0095287460001,"p90_ttft":0.1093862688169,"mean_e2el":"6.8078120582068005","mean_tpot":"0.0072468528808999995","mean_ttft":0.1115084703197,"p50_intvty":144.4107881321,"p90_intvty":104.9456035443,"mean_intvty":137.9909343311},{"ep":2,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":4910480.753864527,"tput_per_gpu":9329.9134323426,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":4706.6578721105,"output_tput_per_gpu":4623.2555602322,"p50_itl":0.728333469131,"p90_itl":1.1271875431295,"mean_itl":0.8278899790976,"p50_e2el":45.3559324195376,"p50_tpot":0.035686458577,"p50_ttft":11.6426672624657,"p90_e2el":73.0168020036538,"p90_tpot":0.054856601034,"p90_ttft":16.1431461957516,"mean_e2el":49.5652248248524,"mean_tpot":0.0403072403666,"mean_ttft":12.4686391525169,"p50_intvty":28.0218334874,"p90_intvty":18.2293467176,"mean_intvty":24.8094384757},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":6143168.280490421,"tput_per_gpu":11672.0197329318,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":10381.2798472147,"output_tput_per_gpu":1290.7398857171,"p50_itl":0.4110635789111,"p90_itl":0.822921329923,"mean_itl":0.4765756609924,"p50_e2el":20.5300070425728,"p50_tpot":0.0215460888866,"p50_ttft":"0.27938759641259997","p90_e2el":31.7589449326741,"p90_tpot":0.0338026730143,"p90_ttft":0.8988701118855,"mean_e2el":22.225353001761,"mean_tpot":"0.023211050706999997","mean_ttft":0.865083026456,"p50_intvty":46.412135644,"p90_intvty":29.58345926,"mean_intvty":43.082926862},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"226287.43215372265","tput_per_gpu":310.0137820506,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":156.7801190038,"output_tput_per_gpu":153.2336630468,"p50_itl":0.1217408906668,"p90_itl":0.1896960616112,"mean_itl":0.128115884746,"p50_e2el":5.600581619888501,"p50_tpot":0.0061441990799,"p50_ttft":0.146030860953,"p90_e2el":7.0905650891364,"p90_tpot":"0.0077858140889999995","p90_ttft":"0.20293485596779998","mean_e2el":5.8602529703639,"mean_tpot":0.0062351643374,"mean_ttft":0.1655021504499,"p50_intvty":162.7551430213,"p90_intvty":128.4387205455,"mean_intvty":160.3806966239},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"308656.84594372264","tput_per_gpu":422.8598789429,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":213.8823601469,"output_tput_per_gpu":208.977518796,"p50_itl":0.1666999962181,"p90_itl":0.2779013548046,"mean_itl":0.1826783138914,"p50_e2el":8.0841780956835,"p50_tpot":0.0083956054077,"p50_ttft":0.1534733111039,"p90_e2el":11.2690334733576,"p90_tpot":0.0114085950212,"p90_ttft":0.340514888987,"mean_e2el":8.433337793313,"mean_tpot":0.0088787382795,"mean_ttft":0.2881687375018,"p50_intvty":119.1099332859,"p90_intvty":87.653212174,"mean_intvty":112.6286155219},{"ep":1,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-03","num_gpus":9,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":349396.6428258421,"tput_per_gpu":663.8536213691,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":5310.6886531341,"output_tput_per_gpu":82.9992423984,"p50_itl":"0.17836537699990002","mean_itl":"0.17564521905140001","p50_e2el":"5.4128639195004995","p50_tpot":0.0055736936489,"p50_ttft":0.2282928010009,"mean_e2el":5.3747696062751,"mean_tpot":0.005575299255,"mean_ttft":0.2503204449503,"p50_intvty":179.4142382025,"mean_intvty":179.3625694808},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":592714.2922567368,"tput_per_gpu":1126.1571552878,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":5005.0106711265,"output_tput_per_gpu":156.4437763281,"p50_itl":0.1916070280131,"mean_itl":"0.18800945964799998","p50_e2el":5.7495760109741,"p50_tpot":"0.0059739449302000005","p50_ttft":0.2152385164518,"mean_e2el":5.730820269818699,"mean_tpot":0.0059681744257,"mean_ttft":0.2456443793199,"p50_intvty":167.3935752131,"mean_intvty":167.555424603},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"1076796.1468549473","tput_per_gpu":2045.9126790244,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":9080.7038778485,"output_tput_per_gpu":287.2148793184,"p50_itl":0.2080638816115,"mean_itl":"0.20441428557320002","p50_e2el":6.3601628096076,"p50_tpot":"0.0064974169911999995","p50_ttft":0.2394146390725,"mean_e2el":6.318027295667,"mean_tpot":0.0064923396933,"mean_ttft":0.3133577437111,"p50_intvty":153.9073144527,"mean_intvty":154.0276768073},{"ep":1,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":64,"date":"2026-07-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":1552388.833501316,"tput_per_gpu":2949.5387836525,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":1491.634519131,"output_tput_per_gpu":1457.9042645215,"p50_itl":0.2585238087922,"p90_itl":0.4051664303057,"mean_itl":"0.43235715432849997","p50_e2el":"12.106127392034999","p50_tpot":0.0128564371859,"p50_ttft":"0.11709394911299999","p90_e2el":50.3444555340335,"p90_tpot":0.0537270693345,"p90_ttft":0.6807086063083,"mean_e2el":19.6870949184413,"mean_tpot":"0.021033169056199997","mean_ttft":0.3911303123925,"p50_intvty":77.7820468873,"p90_intvty":18.6125916114,"mean_intvty":47.5439529502},{"ep":1,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":2,"date":"2026-07-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":283841.6038554737,"tput_per_gpu":539.2990473254,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":267.6733824013,"output_tput_per_gpu":271.6256649242,"p50_itl":0.0684900286142,"p90_itl":0.1009544139728,"mean_itl":0.072724464113,"p50_e2el":"3.2155426931566997","p50_tpot":0.003302929135,"p50_ttft":0.0636922989506,"p90_e2el":4.0902569402475,"p90_tpot":0.0044530717103,"p90_ttft":0.0883482319303,"mean_e2el":3.4211774900788,"mean_tpot":"0.0035332336831000004","mean_ttft":0.1122144904453,"p50_intvty":302.7615668178,"p90_intvty":224.5640908221,"mean_intvty":283.0268500979},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":428109.615504,"tput_per_gpu":813.4082694576,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":722.9204172254,"output_tput_per_gpu":90.4878522321,"p50_itl":"0.048094120982600005","p90_itl":0.0904362272006,"mean_itl":0.0540162499016,"p50_e2el":2.2765202485025,"p50_tpot":0.0022540493351,"p50_ttft":0.1525874404906,"p90_e2el":3.6684600530978,"p90_tpot":0.0038894782712,"p90_ttft":0.1657196877,"mean_e2el":2.5699896609964,"mean_tpot":0.0026081584541,"mean_ttft":0.1554633843945,"p50_intvty":443.6460127312,"p90_intvty":257.1038916461,"mean_intvty":383.4122878715},{"ep":1,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-29","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"2538455.3998045265","tput_per_gpu":4823.0652596286,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":4282.308789991,"output_tput_per_gpu":540.7564696376,"p50_itl":"0.10925342515110001","p90_itl":0.2585815296508,"mean_itl":0.1406631325027,"p50_e2el":5.965767402434699,"p50_tpot":0.0063068239926,"p50_ttft":"0.22206924180499998","p90_e2el":9.7025526224636,"p90_tpot":0.0099546777123,"p90_ttft":"0.38769367821510003","mean_e2el":6.6101777340693,"mean_tpot":0.0068100641703,"mean_ttft":0.2908868057479,"p50_intvty":158.5584124705,"p90_intvty":100.4552863392,"mean_intvty":146.8414944404},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":7780171.902698631,"tput_per_gpu":14782.3266151274,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":13142.8279136643,"output_tput_per_gpu":1639.4987014631,"p50_itl":0.9765073340386,"p90_itl":"3.0602209202479997","mean_itl":1.3914886670075,"p50_e2el":17.50705184252,"p50_tpot":0.0175432219725,"p50_ttft":1.5404705271358,"p90_e2el":"22.080996382329598","p90_tpot":0.0205287801412,"p90_ttft":"2.5612190366489997","mean_e2el":17.6804614629098,"mean_tpot":0.0171984102224,"mean_ttft":1.832710266424,"p50_intvty":57.0020718867,"p90_intvty":48.7121004326,"mean_intvty":58.1449091556},{"ep":1,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-07-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"160452.37481942106","tput_per_gpu":304.8595121569,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":154.1735024787,"output_tput_per_gpu":150.6860096782,"p50_itl":"0.056596839975100004","p90_itl":0.0885787180508,"mean_itl":0.06338392416500001,"p50_e2el":2.792970567476,"p50_tpot":0.0029358469942,"p50_ttft":0.0448056985042,"p90_e2el":3.7931623479817,"p90_tpot":0.0042378284589,"p90_ttft":0.0949432009016,"mean_e2el":2.9204061855533,"mean_tpot":0.0030985762955,"mean_ttft":0.0966524812509,"p50_intvty":340.6172058602,"p90_intvty":235.9699099894,"mean_intvty":322.7288614659},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"464012.89977992704","tput_per_gpu":635.6976726985,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":320.411960876,"output_tput_per_gpu":315.2857118225,"p50_itl":0.2274928428233,"p90_itl":0.3721059700474,"mean_itl":0.247650940509,"p50_e2el":10.8395629124716,"p50_tpot":0.0115289502937,"p50_ttft":0.1717239199206,"p90_e2el":14.8862304287031,"p90_tpot":"0.015814361417800002","p90_ttft":0.3672379070893,"mean_e2el":11.3561143052881,"mean_tpot":0.012066193705,"mean_ttft":0.2644480105489,"p50_intvty":86.7381656201,"p90_intvty":63.2336629711,"mean_intvty":82.8761765682},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":633092.6244513139,"tput_per_gpu":867.3368954983,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":436.2853335184,"output_tput_per_gpu":431.0515619799,"p50_itl":0.3426201120019,"p90_itl":0.5432974848896001,"mean_itl":0.3629657776107,"p50_e2el":15.8158534085378,"p50_tpot":0.0171720563987,"p50_ttft":0.1956011848524,"p90_e2el":22.4154028445482,"p90_tpot":0.0237625641841,"p90_ttft":0.3851421944797,"mean_e2el":16.6696507231274,"mean_tpot":0.0176359378845,"mean_ttft":0.3486678586283,"p50_intvty":58.234143703,"p90_intvty":42.0830004814,"mean_intvty":56.7023997561},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":128,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":6766834.063651264,"tput_per_gpu":12856.9847209374,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":57167.6048990748,"output_tput_per_gpu":1779.329676403,"p50_itl":"0.44663390098140004","mean_itl":"0.43268297380590004","p50_e2el":15.6349003349897,"p50_tpot":0.0139615089116,"p50_ttft":2.7583777974942,"mean_e2el":15.9930348746973,"mean_tpot":0.0137472629064,"mean_ttft":3.3590301513645,"p50_intvty":71.6254959496,"mean_intvty":72.7417527991},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-03","num_gpus":6,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":8846593.562594527,"tput_per_gpu":16808.5277689296,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":44823.4490753546,"output_tput_per_gpu":2801.0671157172,"p50_itl":0.6678273095749,"mean_itl":0.6510823114127999,"p50_e2el":19.9886890819762,"p50_tpot":0.0211142794219,"p50_ttft":0.4521596624982,"mean_e2el":20.3863323516922,"mean_tpot":"0.020669079474699998","mean_ttft":1.3508220087598,"p50_intvty":47.3613131672,"mean_intvty":48.3814482994},{"ep":1,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-29","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":4668772.321894684,"tput_per_gpu":8870.6674115999,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":7882.7037063317,"output_tput_per_gpu":987.9637052682,"p50_itl":0.2556639130053,"p90_itl":0.5295875535085001,"mean_itl":0.3128661091099,"p50_e2el":13.8689041024918,"p50_tpot":0.0144993296485,"p50_ttft":0.2337206204975,"p90_e2el":"19.744952636011302","p90_tpot":0.0206210555748,"p90_ttft":0.552840076093,"mean_e2el":14.6471138534132,"mean_tpot":0.0153011585268,"mean_ttft":0.5231646680907001,"p50_intvty":68.9687057433,"p90_intvty":48.4941227364,"mean_intvty":65.3545284332},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":128,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":7378817.569681052,"tput_per_gpu":14019.753382394,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":12468.6090060667,"output_tput_per_gpu":1551.1443763274,"p50_itl":0.7427383398171999,"p90_itl":1.3012008849531,"mean_itl":0.7989810666736,"p50_e2el":35.5258337795967,"p50_tpot":0.0371367984867,"p50_ttft":0.4397349974606,"p90_e2el":50.2219498144463,"p90_tpot":0.0522883300667,"p90_ttft":"1.0548148811562001","mean_e2el":37.3062688846194,"mean_tpot":"0.039001024377599995","mean_ttft":1.4713440217927,"p50_intvty":26.927469269,"p90_intvty":19.1247262769,"mean_intvty":25.6403521692},{"ep":1,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-07-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":5449527.70261779,"tput_per_gpu":10354.1026349738,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":5223.3302086816,"output_tput_per_gpu":5130.7724262922,"p50_itl":0.8644787638914,"p90_itl":"1.0482160711194999","mean_itl":0.9409920207772,"p50_e2el":41.3937430975493,"p50_tpot":0.0429588137648,"p50_ttft":0.6442798324861,"p90_e2el":69.006968006213,"p90_tpot":0.0548827506037,"p90_ttft":4.5767070754431,"mean_e2el":44.9577471654935,"mean_tpot":0.0468202560959,"mean_ttft":1.8338473005448,"p50_intvty":23.2781101796,"p90_intvty":18.2206611185,"mean_intvty":21.3582770234},{"ep":2,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"4167138.9024931053","tput_per_gpu":7917.5639147369,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":3994.4872556798,"output_tput_per_gpu":3923.076659057,"p50_itl":0.5304610184394001,"p90_itl":0.8251452775672,"mean_itl":0.6377348783322,"p50_e2el":"24.660210461588598","p50_tpot":0.0264322274013,"p50_ttft":"0.24642446427609999","p90_e2el":"47.445927999261094","p90_tpot":0.0447097504865,"p90_ttft":0.6022770617623,"mean_e2el":29.192920895751,"mean_tpot":0.0310180010372,"mean_ttft":0.6273807255793,"p50_intvty":37.8326042985,"p90_intvty":22.3664858139,"mean_intvty":32.2393438184},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":4507079.352545789,"tput_per_gpu":8563.450769837,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":7609.7030798507,"output_tput_per_gpu":953.7476899862,"p50_itl":0.2764961021021,"p90_itl":0.5820451005828,"mean_itl":0.3214369532129,"p50_e2el":"13.058894061483398","p50_tpot":0.0136194860867,"p50_ttft":0.2524795159698,"p90_e2el":23.3021728916559,"p90_tpot":0.0243192168768,"p90_ttft":0.5782461978495,"mean_e2el":"14.842685326433301","mean_tpot":"0.015474267726100001","mean_ttft":0.5497903852869,"p50_intvty":73.4242095212,"p90_intvty":41.1197451409,"mean_intvty":64.6234133788},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1316939.0274272105","tput_per_gpu":2502.1841521117,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":2223.9125074372,"output_tput_per_gpu":278.2716446746,"p50_itl":0.1899412034545,"p90_itl":0.5788378432393,"mean_itl":"0.26655206817189997","p50_e2el":3.2623535091989,"p50_tpot":0.0032596615743,"p50_ttft":0.2577714282088,"p90_e2el":3.5993900268804,"p90_tpot":0.0035313095595,"p90_ttft":0.4188071948476,"mean_e2el":"3.2427447278751003","mean_tpot":0.0032037635322,"mean_ttft":0.2973446046002,"p50_intvty":306.7803135995,"p90_intvty":283.1810644604,"mean_intvty":312.1328993056},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"3104504.9865488946","tput_per_gpu":5898.5594744429,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":5235.6420003663,"output_tput_per_gpu":662.9174740766,"p50_itl":0.3070461555035,"p90_itl":0.9860939435894001,"mean_itl":0.4450977619587,"p50_e2el":5.4196959029941,"p50_tpot":0.0055549588801,"p50_ttft":0.3416801800195,"p90_e2el":6.5703309194068,"p90_tpot":"0.0061677740993999995","p90_ttft":0.5786744616984,"mean_e2el":"5.4535257652263995","mean_tpot":0.0053998091998,"mean_ttft":0.4461761527993,"p50_intvty":180.0193343618,"p90_intvty":162.1330457125,"mean_intvty":185.1917286333},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":4,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2169425.1112646842","tput_per_gpu":4121.9077114029,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":3663.5041853951,"output_tput_per_gpu":458.4035260078,"p50_itl":0.2242089600768,"p90_itl":0.7251087985584,"mean_itl":0.3259542044416,"p50_e2el":3.9816020934959,"p50_tpot":0.0039378662036,"p50_ttft":0.3025043049711,"p90_e2el":4.3698760180851,"p90_tpot":0.0042501680811,"p90_ttft":0.5276755547499999,"mean_e2el":3.9243017481203,"mean_tpot":"0.0039061110067999997","mean_ttft":0.3388058089971,"p50_intvty":253.9446360761,"p90_intvty":235.2848124897,"mean_intvty":256.0091093848},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":1924893.214285737,"tput_per_gpu":3657.2971071429,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":16261.5575270669,"output_tput_per_gpu":506.2320021619,"p50_itl":0.2328742349055,"mean_itl":0.22848472919750001,"p50_e2el":6.9995384019567,"p50_tpot":"0.0072794697495999995","p50_ttft":0.2559295115061,"mean_e2el":7.0050544524609,"mean_tpot":0.0072526454741,"mean_ttft":0.3789973901701,"p50_intvty":137.3726431181,"mean_intvty":137.8807227747},{"ep":2,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":3474342.366206158,"tput_per_gpu":6601.2504957917,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":3337.6466944326,"output_tput_per_gpu":3263.6038013591,"p50_itl":0.3580663129687,"p90_itl":0.5622624469688,"mean_itl":0.3809956068318,"p50_e2el":16.4851505940314,"p50_tpot":0.0175918912421,"p50_ttft":0.1668793399585,"p90_e2el":23.2503733819816,"p90_tpot":0.0247621449235,"p90_ttft":0.2829879245488,"mean_e2el":17.395716643036,"mean_tpot":0.0185375052282,"mean_ttft":"0.37414267829139997","p50_intvty":56.8443714343,"p90_intvty":40.3842237047,"mean_intvty":53.9446914615},{"ep":1,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-07-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":99188.82884221053,"tput_per_gpu":188.4587748002,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":93.538822188,"output_tput_per_gpu":94.9199526121,"p50_itl":"0.049176217056799996","p90_itl":0.0702805303736,"mean_itl":0.0525050854653,"p50_e2el":2.3895931504667,"p50_tpot":0.0024206317684,"p50_ttft":0.0430494335014,"p90_e2el":2.9571145204944,"p90_tpot":0.0028523644062,"p90_ttft":0.0486951844301,"mean_e2el":2.4652681525622,"mean_tpot":0.0025450784072,"mean_ttft":0.0815373649471,"p50_intvty":413.1152920639,"p90_intvty":350.5863408765,"mean_intvty":392.9152033919},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":1536,"date":"2026-07-30","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"10016013.780024858","tput_per_gpu":21233.9492136527,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":18,"input_tput_per_gpu":35649.0381292083,"output_tput_per_gpu":5016.9741836526,"p50_itl":3.1651531190146,"mean_itl":2.7262007701561,"p50_e2el":17.010709334485,"p50_tpot":0.0110377130672,"p50_ttft":7.0750901300053,"mean_e2el":"17.126151157264403","mean_tpot":0.0108892808902,"mean_ttft":7.0938078711897,"p50_intvty":90.5984775935,"mean_intvty":91.8334286791},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":90,"date":"2026-07-30","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1175319.5641411794","tput_per_gpu":2491.6774759793,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":60,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":35450.4072087714,"output_tput_per_gpu":294.4288271265,"p50_itl":0.9123719939962,"mean_itl":0.7866333039761999,"p50_e2el":4.4366246930149,"p50_tpot":0.0031683500617,"p50_ttft":1.5241092669894,"mean_e2el":4.564161975134399,"mean_tpot":0.0031390024829,"mean_ttft":1.6792937172508,"p50_intvty":315.6216896888,"mean_intvty":318.5725418976},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":48,"date":"2026-07-30","num_gpus":18,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"2035862.6274417455","tput_per_gpu":4316.0287701765,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":34543.8375146409,"output_tput_per_gpu":537.5526771185,"p50_itl":"1.0889078034961002","mean_itl":0.943381922509,"p50_e2el":4.902269474521701,"p50_tpot":0.0038067211378,"p50_ttft":"1.3700259679753999","mean_e2el":5.0160911601916,"mean_tpot":"0.0037773837831999997","mean_ttft":1.544740614792,"p50_intvty":262.693263783,"mean_intvty":264.7334921155},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":168,"date":"2026-07-30","num_gpus":62,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1812114.9253559907","tput_per_gpu":3841.6836417547,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":35283.8084815362,"output_tput_per_gpu":472.8845517782,"p50_itl":1.0519453589804,"mean_itl":0.9039738610217001,"p50_e2el":5.5715910019935,"p50_tpot":0.0036537203455,"p50_ttft":2.2231095839816,"mean_e2el":5.6350106622007,"mean_tpot":0.0036116603886,"mean_ttft":2.3079968478375,"p50_intvty":273.693634277,"mean_intvty":276.8809612182},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":384,"date":"2026-07-30","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":6281898.550334623,"tput_per_gpu":13317.6249267094,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":35515.8480809257,"output_tput_per_gpu":2218.5133496013,"p50_itl":1.7410581090371,"mean_itl":"1.4992524266370002","p50_e2el":9.5450866429892,"p50_tpot":0.0060704183109,"p50_ttft":4.0280418315087,"mean_e2el":9.6287801986154,"mean_tpot":0.0060045199694,"mean_ttft":4.1005310010496,"p50_intvty":164.7332932906,"mean_intvty":166.5412064744},{"ep":1,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-30","num_gpus":66,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":95928.17036924529,"tput_per_gpu":203.3677211828,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":64,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":5955.6648088464,"output_tput_per_gpu":23.6084371933,"p50_itl":0.6508948060218,"mean_itl":0.5685635874687001,"p50_e2el":2.3931389609934,"p50_tpot":0.0022988643516,"p50_ttft":0.2425401924702,"mean_e2el":2.4065826944197,"mean_tpot":0.0022908299807,"mean_ttft":"0.28335321346910003","p50_intvty":434.9973930932,"mean_intvty":436.5230106183},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":3056014.131874421,"tput_per_gpu":5806.4268505614,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":25788.2622982264,"output_tput_per_gpu":810.9679886451,"p50_itl":0.2892033749959,"mean_itl":0.2830215775159,"p50_e2el":8.712319772515901,"p50_tpot":0.009040432213800001,"p50_ttft":0.3201398519741,"mean_e2el":8.8392394871666,"mean_tpot":0.008989160368400001,"mean_ttft":0.5299029346594,"p50_intvty":110.6141804233,"mean_intvty":111.2450950943},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":480,"date":"2026-07-30","num_gpus":52,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":4346029.913455944,"tput_per_gpu":9213.5834165266,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":35491.2436168404,"output_tput_per_gpu":1330.2853564324,"p50_itl":1.5220715310134,"mean_itl":1.3081861913221,"p50_e2el":7.9110792610154,"p50_tpot":0.0052792813598,"p50_ttft":3.1034543934802,"mean_e2el":8.0369609655286,"mean_tpot":0.005210250506,"mean_ttft":3.2395970265064,"p50_intvty":189.4197205729,"mean_intvty":191.9293513507},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":42,"date":"2026-07-30","num_gpus":58,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":641362.3011633019,"tput_per_gpu":1359.6880784662,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":35061.5833744068,"output_tput_per_gpu":156.048960754,"p50_itl":0.8302873549983,"mean_itl":0.7177594403548,"p50_e2el":4.1927020669973,"p50_tpot":0.0029000403765,"p50_ttft":"1.5296127694891999","mean_e2el":4.2823534872895,"mean_tpot":0.0028713664845,"mean_ttft":1.6409561115802,"p50_intvty":344.8227852606,"mean_intvty":348.2662367868},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":192,"date":"2026-07-30","num_gpus":38,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"2981485.7440465093","tput_per_gpu":6320.7497773786,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":35582.1606750644,"output_tput_per_gpu":834.2352340625,"p50_itl":1.2113214879937,"mean_itl":1.0415574053059,"p50_e2el":6.3641238840064,"p50_tpot":0.0041972869374,"p50_ttft":2.5220369359886,"mean_e2el":6.4388672707173,"mean_tpot":0.0041478030577,"mean_ttft":2.6198235484087,"p50_intvty":238.249139244,"mean_intvty":241.0914853212},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":160,"date":"2026-03-23","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":824974.5135229927,"tput_per_gpu":1130.2150835265,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":2679.9624845299,"output_tput_per_gpu":200.3666429244,"p50_itl":3.9621351920068,"mean_itl":3.432493078043,"p50_e2el":"17.796283358940897","p50_tpot":0.0161986965615,"p50_ttft":3.0017717814771,"mean_e2el":"17.620267784017802","mean_tpot":0.0151711563269,"mean_ttft":3.6779101638748,"p50_intvty":61.7333620766,"mean_intvty":65.9145538054},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":512,"date":"2026-06-03","num_gpus":6,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"11043272.311915578","tput_per_gpu":20982.2173926396,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":55956.6274539481,"output_tput_per_gpu":3495.0123619854,"p50_itl":1.0426559259649,"mean_itl":0.9991397677725,"p50_e2el":32.5871270535281,"p50_tpot":0.0333859396369,"p50_ttft":1.2684790894855,"mean_e2el":32.8995826457392,"mean_tpot":0.0317307722439,"mean_ttft":3.6759153839706,"p50_intvty":29.9527289295,"mean_intvty":31.5151485225},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-03","num_gpus":6,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":4252159.624367316,"tput_per_gpu":8079.1032862979,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":21550.5568176221,"output_tput_per_gpu":1343.3765206357,"p50_itl":0.3560889465734,"mean_itl":0.3481319445468,"p50_e2el":10.5676364348037,"p50_tpot":0.0111332953461,"p50_ttft":0.2558715546038,"mean_e2el":10.6737646839258,"mean_tpot":0.0110538919422,"mean_ttft":0.4985207849837,"p50_intvty":89.8206657523,"mean_intvty":90.4658743933},{"dp":1,"hw":"H200","pp":1,"tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":9888451.503649635,"dataset_name":"random","tput_per_gpu":13547.17856,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":108.8462807,"input_tput_per_gpu":12041.9365,"total_input_tokens":2621440,"output_tput_per_gpu":1505.242063,"total_output_tokens":327680,"tput_requests_per_sec":2.939925904,"p50_e2el":21.64914323,"p50_tpot":"0.020381063999999997","p50_ttft":0.7993146809999999,"p50_intvty":49.06515167},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":250793.8269343066,"dataset_name":"random","tput_per_gpu":343.5875429,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":16.76428648,"input_tput_per_gpu":305.4111493,"total_input_tokens":40960,"output_tput_per_gpu":38.17639366,"total_output_tokens":5120,"tput_requests_per_sec":0.298253075,"p50_e2el":3.353059788,"p50_tpot":"0.0031966480000000003","p50_ttft":0.082888682,"p50_intvty":312.8276677},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":471869.4991240876,"dataset_name":"random","tput_per_gpu":646.4612138,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":17.82009462,"input_tput_per_gpu":574.63219,"total_input_tokens":81920,"output_tput_per_gpu":71.82902376,"total_output_tokens":10240,"tput_requests_per_sec":0.561164248,"p50_e2el":3.560005599,"p50_tpot":"0.0033730310000000003","p50_ttft":0.109395226,"p50_intvty":296.4692879},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":190757.1469566842,"tput_per_gpu":362.4385792177,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":180.6457295728,"output_tput_per_gpu":181.7928496449,"p50_itl":2.3277738749748,"p90_itl":"2.5943775309715003","mean_itl":2.2016222370773,"p50_e2el":9.4568290154566,"p50_tpot":0.0100542820773,"p50_ttft":0.1602477750275,"p90_e2el":"12.448685503017598","p90_tpot":"0.013336146039200001","p90_ttft":0.5262353247148,"mean_e2el":9.9405291019517,"mean_tpot":"0.010422230130999998","mean_ttft":0.2809118332749,"p50_intvty":99.4601098626,"p90_intvty":74.9841818665,"mean_intvty":95.9487544821},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":41531.70073373684,"tput_per_gpu":78.9102313941,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":39.3370058966,"output_tput_per_gpu":39.5732254975,"p50_itl":"1.3124679725005999","p90_itl":1.3313824329089,"mean_itl":1.1774957707722,"p50_e2el":5.9013522439927,"p50_tpot":0.0057952412948,"p50_ttft":0.1172301069892,"p90_e2el":7.4866173477756,"p90_tpot":"0.0065760190121999995","p90_ttft":1.6904430321913,"mean_e2el":5.8204203886911,"mean_tpot":0.0056148672985,"mean_ttft":0.6394392232964,"p50_intvty":172.555368987,"p90_intvty":152.0676868694,"mean_intvty":178.098599102},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":12,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":81307.33957839622,"tput_per_gpu":172.3715599062,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":1472.7323262995,"output_tput_per_gpu":91.0990120066,"p50_itl":0.8782400769995999,"mean_itl":0.7781435256088001,"p50_e2el":3.7379440465011,"p50_tpot":0.003728551464,"p50_ttft":0.3578497510171,"mean_e2el":3.6398164080199,"mean_tpot":0.0035723485751,"mean_ttft":0.3716137303301,"p50_intvty":268.200669794,"mean_intvty":279.9278902858},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":28,"date":"2026-03-23","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"272436.65092766425","tput_per_gpu":373.2382117709,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2653.8234255888,"output_tput_per_gpu":47.4403240827,"p50_itl":2.0654048461000998,"mean_itl":1.7797130514927,"p50_e2el":9.3852751390077,"p50_tpot":0.0085436350249,"p50_ttft":1.5596411650768,"mean_e2el":9.3035170864106,"mean_tpot":0.0080490565104,"mean_ttft":1.8922836605484,"p50_intvty":117.046198379,"mean_intvty":124.2381636542},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":120,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"397226.95825142105","tput_per_gpu":754.7312206777,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":7939.809450794,"output_tput_per_gpu":395.4773091719,"p50_itl":1.7068251119927,"mean_itl":1.504208827984,"p50_e2el":"6.8639008614699994","p50_tpot":0.0072367782108,"p50_ttft":0.1890602840576,"mean_e2el":6.6649840745679,"mean_tpot":0.0069179351588,"mean_ttft":0.315969554947,"p50_intvty":138.1830382072,"mean_intvty":144.5518029649},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-30","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"184235.59157226316","tput_per_gpu":350.0476239873,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":2174.9080111697,"output_tput_per_gpu":45.9042261236,"p50_itl":0.0144105819636,"mean_itl":0.0141727379751,"p50_e2el":5.9701599565451,"p50_tpot":0.0052865521802,"p50_ttft":0.7497390829958,"mean_e2el":6.4927352186627,"mean_tpot":0.0054373437784,"mean_ttft":1.4624762813357,"p50_intvty":189.159203563,"mean_intvty":183.9133298825},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":7746494.671532848,"dataset_name":"random","tput_per_gpu":10612.6977,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":69.47149731,"input_tput_per_gpu":9433.50907,"total_input_tokens":2621440,"output_tput_per_gpu":1179.188634,"total_output_tokens":327680,"tput_requests_per_sec":4.606205601,"p50_e2el":14.06502454,"p50_tpot":0.013194243,"p50_ttft":0.567314194,"p50_intvty":75.79063218},{"dp":1,"hw":"H200","pp":1,"tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":9456430.175182482,"dataset_name":"random","tput_per_gpu":12955.30934,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":202.3448403,"input_tput_per_gpu":6477.654671,"total_input_tokens":2621440,"output_tput_per_gpu":6477.654671,"total_output_tokens":2621440,"tput_requests_per_sec":12.65166928,"p50_e2el":39.91009949,"p50_tpot":0.038500541,"p50_ttft":0.524046413,"p50_intvty":25.97366124},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"205050.48252591243","tput_per_gpu":280.9191610605,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":66.8374425706},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":666,"date":"2026-02-03","num_gpus":36,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":5745264.126273821,"tput_per_gpu":12179.9599477005,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":20,"input_tput_per_gpu":19489.4672037892,"output_tput_per_gpu":3043.0758775896,"p50_itl":2.5447164129873,"mean_itl":2.2056625931905,"p50_e2el":12.1429532180191,"p50_tpot":"0.010320322778299999","p50_ttft":"2.7212235445040003","mean_e2el":11.8944292825125,"mean_tpot":0.0095917966742,"mean_ttft":"3.0691295953088003","p50_intvty":96.8961941873,"mean_intvty":104.2557545755},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":8192,"osl":1024,"conc":180,"date":"2026-02-03","num_gpus":40,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":2231514.7829075,"tput_per_gpu":4730.8113397639,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":21030.8934815715,"output_tput_per_gpu":655.790804312,"p50_itl":1.4644192999986,"mean_itl":1.2775204831784,"p50_e2el":7.503871290508,"p50_tpot":0.0059842742751,"p50_ttft":"2.1143423394823997","mean_e2el":7.3823316377427,"mean_tpot":"0.0055251656252999996","mean_ttft":2.3020586394334,"p50_intvty":167.1046402675,"mean_intvty":180.9900494967},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":5,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":33725.73300273585,"tput_per_gpu":71.4985539658,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":608.9784318577,"output_tput_per_gpu":37.9060615975,"p50_itl":0.7575018520001,"mean_itl":0.6646102177453,"p50_e2el":3.7866537709779,"p50_tpot":0.0032449972976,"p50_ttft":0.537719629996,"mean_e2el":3.7384476805502,"mean_tpot":0.0030970610893,"mean_ttft":0.8806239112839,"p50_intvty":308.1666665028,"mean_intvty":322.8867533353},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":11,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"39475.851645421055","tput_per_gpu":75.0041181263,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":789.0391299167,"output_tput_per_gpu":39.3023675368,"p50_itl":0.7440155820223,"mean_itl":0.6618175429718,"p50_e2el":6.5864598939661,"p50_tpot":"0.0031857889095999996","p50_ttft":3.6856998261064,"mean_e2el":6.123232413671699,"mean_tpot":"0.0030864185010999997","mean_ttft":3.2834338487274,"p50_intvty":313.8939924692,"mean_intvty":324.0001314305},{"hw":"H200","isl":8192,"osl":1024,"conc":64,"date":"2026-10-16","sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","tput_per_mw":"1096661.5240433577","tput_per_gpu":1502.4262879394,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1792134000,"serving_config":"disagg","total_requests":640,"input_tput_per_gpu":2671.5305249607,"output_tput_per_gpu":333.3220509182,"p50_itl":4.2312404480763,"p90_itl":4.2509133289568,"mean_itl":3.7196868699648,"p50_e2el":"20.842199738370297","p50_tpot":0.0189690709472,"p50_ttft":3.6001777495258,"p90_e2el":24.1937452983111,"p90_tpot":0.0204839276039,"p90_ttft":4.9480729513336,"mean_e2el":21.1218425950778,"mean_tpot":0.0183198022281,"mean_ttft":4.2611996898399,"p50_intvty":52.7173946885,"p90_intvty":48.818762658,"mean_intvty":54.5857421139},{"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-04","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":174425.0148681579,"tput_per_gpu":331.4075282495,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":171.1672597871},{"hw":"B300","tp":1,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-04","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":923313.0848602105,"tput_per_gpu":1754.2948612344,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":113.0023458725},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"50906.741010218975","tput_per_gpu":69.742235184,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":72.634450126},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-24","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":774867.4462340936,"tput_per_gpu":1325.0233330603,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":179.948624945},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":5813762.909489051,"dataset_name":"random","tput_per_gpu":7964.855186,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":46.28332737,"input_tput_per_gpu":7079.871276,"total_input_tokens":1310720,"output_tput_per_gpu":884.9839095,"total_output_tokens":163840,"tput_requests_per_sec":3.456968397,"p50_e2el":9.233065663,"p50_tpot":0.008478001,"p50_ttft":0.5600711979999999,"p50_intvty":117.9523409},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"15073755.226277374","dataset_name":"random","tput_per_gpu":20651.04466,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":1142.458427,"input_tput_per_gpu":18356.48414,"total_input_tokens":20971520,"output_tput_per_gpu":2294.560518,"total_output_tokens":2621440,"tput_requests_per_sec":2.240781756,"p50_e2el":236.8557691,"p50_tpot":0.078611908,"p50_ttft":156.4357867,"p50_intvty":12.72071904},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":14974979.80291971,"dataset_name":"random","tput_per_gpu":20515.72233,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":574.9970589,"input_tput_per_gpu":18236.19762,"total_input_tokens":10485760,"output_tput_per_gpu":2279.524703,"total_output_tokens":1310720,"tput_requests_per_sec":2.226098343,"p50_e2el":99.04666932,"p50_tpot":0.07759542500000001,"p50_ttft":19.6665494,"p50_intvty":12.8873577},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":12396462.81751825,"dataset_name":"random","tput_per_gpu":16983.15406,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":173.6497231,"input_tput_per_gpu":15096.13694,"total_input_tokens":2621440,"output_tput_per_gpu":1887.017118,"total_output_tokens":327680,"tput_requests_per_sec":1.842790154,"p50_e2el":"34.664769740000004","p50_tpot":0.032848843,"p50_ttft":1.060403298,"p50_intvty":30.44247246},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"1476448.9583941607","dataset_name":"random","tput_per_gpu":2022.735073,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":22.78103574,"input_tput_per_gpu":1797.986732,"total_input_tokens":327680,"output_tput_per_gpu":224.7483415,"total_output_tokens":40960,"tput_requests_per_sec":1.755846418,"p50_e2el":"4.5500656799999994","p50_tpot":0.004138305,"p50_ttft":"0.31657984699999997","p50_intvty":241.6448384},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"3753861.2408759124","dataset_name":"random","tput_per_gpu":5142.7899,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":35.84046862,"input_tput_per_gpu":4571.3688,"total_input_tokens":655360,"output_tput_per_gpu":571.4211,"total_output_tokens":81920,"tput_requests_per_sec":2.232113672,"p50_e2el":7.171786936,"p50_tpot":0.006475682,"p50_ttft":0.5471638249999999,"p50_intvty":154.4238794},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":32,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":9721311.693430657,"dataset_name":"random","tput_per_gpu":13318.19702,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":110.7176894,"input_tput_per_gpu":11838.39735,"total_input_tokens":1310720,"output_tput_per_gpu":1479.799668,"total_output_tokens":163840,"tput_requests_per_sec":1.445116864,"p50_e2el":21.67807871,"p50_tpot":0.020152486,"p50_ttft":1.062086034,"p50_intvty":49.62167072},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":863670.0226277373,"dataset_name":"random","tput_per_gpu":1183.227931,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":19.47215697,"input_tput_per_gpu":1051.758161,"total_input_tokens":81920,"output_tput_per_gpu":131.4697701,"total_output_tokens":10240,"tput_requests_per_sec":0.51355379,"p50_e2el":3.890964555,"p50_tpot":0.003665886,"p50_ttft":0.140763429,"p50_intvty":272.7853695},{"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-04","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"2579645.1940707364","tput_per_gpu":4901.3258687344,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":79.0570177071},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":2398605.535766423,"dataset_name":"random","tput_per_gpu":3286.089584,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":28.04549226,"input_tput_per_gpu":2920.968519,"total_input_tokens":655360,"output_tput_per_gpu":365.1210649,"total_output_tokens":81920,"tput_requests_per_sec":2.85250832,"p50_e2el":5.594319984,"p50_tpot":0.004976116,"p50_ttft":0.50375361,"p50_intvty":200.9599571},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":59796.71977372263,"dataset_name":"random","tput_per_gpu":81.92150609,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":14.05918977,"input_tput_per_gpu":42.27661835,"total_input_tokens":4755,"output_tput_per_gpu":39.64488774,"total_output_tokens":4459,"tput_requests_per_sec":0.355639271,"p50_e2el":3.225472636,"p50_tpot":0.003131095,"p50_ttft":"0.022362628000000002","p50_intvty":319.3771046},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"1047892.8678832118","dataset_name":"random","tput_per_gpu":1435.613229,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":28.5313615,"input_tput_per_gpu":717.8066144,"total_input_tokens":81920,"output_tput_per_gpu":717.8066144,"total_output_tokens":81920,"tput_requests_per_sec":2.803932087,"p50_e2el":5.704254134,"p50_tpot":0.005373723,"p50_ttft":"0.20693603200000002","p50_intvty":186.0907412},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"363594.04182481754","dataset_name":"random","tput_per_gpu":498.1238373,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":20.55713707,"input_tput_per_gpu":249.0619187,"total_input_tokens":40960,"output_tput_per_gpu":249.0619187,"total_output_tokens":40960,"tput_requests_per_sec":1.94579624,"p50_e2el":4.109740406,"p50_tpot":0.003927476,"p50_ttft":0.09193291,"p50_intvty":254.6164795},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"32276833.058479536","tput_per_gpu":55193.38453,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":192.6989,"input_tput_per_gpu":49082.91,"kv_cache_precision":"FP8","total_input_tokens":9458221,"output_tput_per_gpu":6110.477,"total_output_tokens":1177482,"tput_requests_per_sec":6.642488,"p50_e2el":"20.678459999999998","p50_tpot":"0.020027999999999997","p50_ttft":0.190048,"p50_intvty":49.93066},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"32409554.555555556","tput_per_gpu":55420.33829,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":192.6989,"input_tput_per_gpu":49082.91,"kv_cache_precision":"FP8","total_input_tokens":9458221,"output_tput_per_gpu":6110.477,"total_output_tokens":1177482,"tput_requests_per_sec":6.642488,"p50_e2el":"20.678459999999998","p50_tpot":"0.020027999999999997","p50_ttft":0.190048,"p50_intvty":49.76625},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":384,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":38147542.61403509,"tput_per_gpu":65232.29787,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":488.5011,"input_tput_per_gpu":57984.14,"kv_cache_precision":"FP8","total_input_tokens":28325314,"output_tput_per_gpu":7248.162,"total_output_tokens":3540735,"tput_requests_per_sec":7.860781,"p50_e2el":50.74934,"p50_tpot":0.039427,"p50_ttft":10.41593,"p50_intvty":25.36359},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":384,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":25475204.43859649,"tput_per_gpu":43562.59959,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":163.0554,"input_tput_per_gpu":21852.86,"kv_cache_precision":"FP8","total_input_tokens":3563226,"output_tput_per_gpu":21709.74,"total_output_tokens":3539890,"tput_requests_per_sec":23.55028,"p50_e2el":"17.088810000000002","p50_tpot":"0.016565999999999997","p50_ttft":"0.14205299999999998","p50_intvty":60.36553},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":640,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"30405119.023391817","tput_per_gpu":51992.75353,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":227.7071,"input_tput_per_gpu":26090.0,"kv_cache_precision":"FP8","total_input_tokens":5940879,"output_tput_per_gpu":25902.75,"total_output_tokens":5898240,"tput_requests_per_sec":28.10628,"p50_e2el":24.15853,"p50_tpot":0.023439,"p50_ttft":0.180157,"p50_intvty":42.66344},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":256,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":5956798.322627738,"dataset_name":"random","tput_per_gpu":8160.813702,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":80.30571753,"input_tput_per_gpu":4080.406851,"total_input_tokens":1310720,"output_tput_per_gpu":4080.406851,"total_output_tokens":1310720,"tput_requests_per_sec":15.93908926,"p50_e2el":"14.846854230000002","p50_tpot":"0.014211600000000001","p50_ttft":"0.30838764300000004","p50_intvty":70.36505491},{"dp":1,"hw":"H200","pp":1,"tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-03-27","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":7535603.058394161,"dataset_name":"random","tput_per_gpu":10323.77619,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":126.9612955,"input_tput_per_gpu":5161.888097,"total_input_tokens":1310720,"output_tput_per_gpu":5161.888097,"total_output_tokens":1310720,"tput_requests_per_sec":10.08181269,"p50_e2el":25.14870601,"p50_tpot":0.024100329,"p50_ttft":"0.49406951400000004","p50_intvty":41.49320961},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":14845451.03508772,"tput_per_gpu":25385.72127,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":52.21762,"input_tput_per_gpu":22550.49,"kv_cache_precision":"FP8","total_input_tokens":2355066,"output_tput_per_gpu":2835.231,"total_output_tokens":296098,"tput_requests_per_sec":6.128199,"p50_e2el":"5.5283050000000005","p50_tpot":"0.0053170000000000005","p50_ttft":0.08912,"p50_intvty":188.0796},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1024,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"35688604.385964915","tput_per_gpu":61027.5135,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":310.4982,"input_tput_per_gpu":30622.66,"kv_cache_precision":"FP8","total_input_tokens":9508283,"output_tput_per_gpu":30404.85,"total_output_tokens":9440652,"tput_requests_per_sec":32.97925,"p50_e2el":33.33883,"p50_tpot":0.032394,"p50_ttft":0.199619,"p50_intvty":30.86977},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"1456150.0894736843","tput_per_gpu":2490.016653,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":18.57066,"input_tput_per_gpu":1251.396,"kv_cache_precision":"FP8","total_input_tokens":92957,"output_tput_per_gpu":1238.621,"total_output_tokens":92008,"tput_requests_per_sec":5.384839,"p50_e2el":"1.9160409999999999","p50_tpot":"0.0018360000000000002","p50_ttft":0.037932,"p50_intvty":544.697},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"19929844.795321636","tput_per_gpu":34080.0346,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":78.10109,"input_tput_per_gpu":30303.83,"kv_cache_precision":"FP8","total_input_tokens":4733525,"output_tput_per_gpu":3776.202,"total_output_tokens":589851,"tput_requests_per_sec":8.194507,"p50_e2el":8.271427,"p50_tpot":0.007989,"p50_ttft":0.098709,"p50_intvty":125.1726},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":8304329.47368421,"tput_per_gpu":14200.4034,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":41.70322,"input_tput_per_gpu":7130.409,"kv_cache_precision":"FP8","total_input_tokens":594722,"output_tput_per_gpu":7069.994,"total_output_tokens":589683,"tput_requests_per_sec":15.34654,"p50_e2el":4.206697,"p50_tpot":0.004071,"p50_ttft":0.041819,"p50_intvty":245.6255},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":4,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":5115655.769590644,"tput_per_gpu":8747.771366,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":23.57523,"input_tput_per_gpu":7775.448,"kv_cache_precision":"FP8","total_input_tokens":733232,"output_tput_per_gpu":972.3234,"total_output_tokens":91691,"tput_requests_per_sec":4.24174,"p50_e2el":2.466698,"p50_tpot":0.002347,"p50_ttft":0.065676,"p50_intvty":426.0686},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-02-05","num_gpus":4,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":4409528.661403509,"tput_per_gpu":7540.294011,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":21.88237,"input_tput_per_gpu":6694.076,"kv_cache_precision":"FP8","total_input_tokens":585929,"output_tput_per_gpu":846.2178,"total_output_tokens":74069,"tput_requests_per_sec":3.655911,"p50_e2el":"2.3138829999999997","p50_tpot":0.002199,"p50_ttft":0.064514,"p50_intvty":454.7942},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"2767628.2128654975","tput_per_gpu":4732.644244,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":17.62566,"input_tput_per_gpu":4204.834,"kv_cache_precision":"FP8","total_input_tokens":74113,"output_tput_per_gpu":527.8099,"total_output_tokens":9303,"tput_requests_per_sec":0.567354,"p50_e2el":1.631976,"p50_tpot":0.001501,"p50_ttft":0.096344,"p50_intvty":666.1751},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":2124084.11871345,"tput_per_gpu":3632.183843,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":20.30837,"input_tput_per_gpu":1830.747,"kv_cache_precision":"FP8","total_input_tokens":148718,"output_tput_per_gpu":1801.437,"total_output_tokens":146337,"tput_requests_per_sec":7.878523,"p50_e2el":2.134473,"p50_tpot":0.002048,"p50_ttft":0.039658,"p50_intvty":488.3487},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":512,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"28498719.292397663","tput_per_gpu":48732.80999,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":194.4037,"input_tput_per_gpu":24448.36,"kv_cache_precision":"FP8","total_input_tokens":4752851,"output_tput_per_gpu":24284.45,"total_output_tokens":4720987,"tput_requests_per_sec":26.33695,"p50_e2el":20.55348,"p50_tpot":0.019942,"p50_ttft":0.15266,"p50_intvty":50.14503},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":32,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":5567275.549122807,"tput_per_gpu":9520.041189,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":31.11536,"input_tput_per_gpu":4762.166,"kv_cache_precision":"FP8","total_input_tokens":296353,"output_tput_per_gpu":4757.875,"total_output_tokens":296086,"tput_requests_per_sec":10.28431,"p50_e2el":3.213565,"p50_tpot":0.003106,"p50_ttft":"0.036338999999999996","p50_intvty":321.979},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":16,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":3530984.303508772,"tput_per_gpu":6037.983159,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":24.43324,"input_tput_per_gpu":3043.354,"kv_cache_precision":"FP8","total_input_tokens":148718,"output_tput_per_gpu":2994.629,"total_output_tokens":146337,"tput_requests_per_sec":6.548456,"p50_e2el":2.610728,"p50_tpot":"0.0025150000000000003","p50_ttft":0.037376,"p50_intvty":397.536},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":7020539.50877193,"tput_per_gpu":12005.12256,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":27.48818,"input_tput_per_gpu":10657.83,"kv_cache_precision":"FP8","total_input_tokens":585929,"output_tput_per_gpu":1347.288,"total_output_tokens":74069,"tput_requests_per_sec":2.910342,"p50_e2el":2.902755,"p50_tpot":0.002764,"p50_ttft":0.075179,"p50_intvty":361.7941},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":8068229.777777779,"tput_per_gpu":13796.67292,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":29.89572,"input_tput_per_gpu":12263.16,"kv_cache_precision":"FP8","total_input_tokens":733232,"output_tput_per_gpu":1533.514,"total_output_tokens":91691,"tput_requests_per_sec":3.34496,"p50_e2el":3.206322,"p50_tpot":0.003058,"p50_ttft":0.077774,"p50_intvty":326.9887},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":5176747.134982993,"tput_per_gpu":7092.1435749267,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":28.0015287379},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":6345135.088827007,"tput_per_gpu":8692.835071693,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":16.5301969423},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":7589655.394160585,"dataset_name":"random","tput_per_gpu":10397.82789,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":126.0570971,"input_tput_per_gpu":5198.913946,"total_input_tokens":2621440,"output_tput_per_gpu":5198.913946,"total_output_tokens":2621440,"tput_requests_per_sec":20.3082576,"p50_e2el":"25.249159329999998","p50_tpot":0.024208873,"p50_ttft":"0.48348214100000003","p50_intvty":41.30716847},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":2988232.65499511,"tput_per_gpu":4093.8787373433,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":65.3300512735},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"15817149.643752575","tput_per_gpu":27047.3258908169,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":14.8658844168},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":5347550.932021872,"tput_per_gpu":9144.3120937574,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":55.0251263419},{"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":41514.91393497076,"tput_per_gpu":70.9905028288,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":234.1828697817},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"212517.76575766082","tput_per_gpu":363.4053794456,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":152.6492332397},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1343004.1471676608","tput_per_gpu":2296.5370916567,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":87.8400096466},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":5520810.32491462,"tput_per_gpu":9440.585655604,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":26.3495185538},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":12,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"11768.867924528302","tput_per_gpu":24.95,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":275.4821},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"11842730.226277372","dataset_name":"random","tput_per_gpu":16224.54041,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":323.1450548,"input_tput_per_gpu":8112.270205,"total_input_tokens":2621440,"output_tput_per_gpu":8112.270205,"total_output_tokens":2621440,"tput_requests_per_sec":7.922138872,"p50_e2el":63.89037131,"p50_tpot":0.061819658,"p50_ttft":0.648861384,"p50_intvty":16.17608437},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":4099404.340875913,"dataset_name":"random","tput_per_gpu":5616.183947,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":58.34566729,"input_tput_per_gpu":2808.091973,"total_input_tokens":655360,"output_tput_per_gpu":2808.091973,"total_output_tokens":655360,"tput_requests_per_sec":10.96910927,"p50_e2el":11.38600505,"p50_tpot":0.010838823,"p50_ttft":"0.29788942500000004","p50_intvty":92.26094265},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":629618.4547445256,"dataset_name":"random","tput_per_gpu":862.577283,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":23.7428001,"input_tput_per_gpu":431.2886415,"total_input_tokens":81920,"output_tput_per_gpu":431.2886415,"total_output_tokens":81920,"tput_requests_per_sec":3.369442512,"p50_e2el":4.736418284,"p50_tpot":0.004496003,"p50_ttft":0.137007163,"p50_intvty":222.4197779},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":2315016.867045497,"tput_per_gpu":3958.6788426478,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":63.0975202207},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1024,"date":"2026-05-06","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":7222541.40468655,"tput_per_gpu":12350.545802014,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":14.8217520114},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"110398.13119567253","tput_per_gpu":188.7808043446,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":175.5825758761},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":740161.0973375438,"tput_per_gpu":1265.6754764472,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":107.0933360307},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":10087.72376380117,"tput_per_gpu":17.2500076361,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":223.6048624267},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":521927.8067543796,"tput_per_gpu":715.0410952535,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":65.7810178646},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":1197,"date":"2026-01-28","num_gpus":26,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":7438891.639831789,"tput_per_gpu":14133.8941156804,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":18,"input_tput_per_gpu":18146.8913219487,"output_tput_per_gpu":5104.6504015766,"p50_itl":"4.3113564939704005","mean_itl":3.8760063863751,"p50_e2el":"25.350019320612798","p50_tpot":0.0235543225688,"p50_ttft":3.7911633964395,"mean_e2el":"25.982359672289302","mean_tpot":0.0231368222057,"mean_ttft":4.6798676670795,"p50_intvty":42.4550524464,"mean_intvty":43.2211472738},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":768,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":32930397.76023392,"tput_per_gpu":56310.98017,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":252.3166,"input_tput_per_gpu":28252.88,"kv_cache_precision":"FP8","total_input_tokens":7128671,"output_tput_per_gpu":28058.1,"total_output_tokens":7079524,"tput_requests_per_sec":30.43795,"p50_e2el":27.14342,"p50_tpot":0.026373,"p50_ttft":0.163626,"p50_intvty":37.91727},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"10393234.701754386","tput_per_gpu":17772.43134,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":37.17187,"input_tput_per_gpu":15804.16,"kv_cache_precision":"FP8","total_input_tokens":1174940,"output_tput_per_gpu":1968.276,"total_output_tokens":146329,"tput_requests_per_sec":4.30433,"p50_e2el":3.962087,"p50_tpot":0.003797,"p50_ttft":0.077889,"p50_intvty":263.3748},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"135619.27275437958","tput_per_gpu":185.7984036735,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":143.5786861195},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":128,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1072795.4803470804","tput_per_gpu":1469.7298080755,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":11.0832931413},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":73527.47475766424,"tput_per_gpu":100.732640418,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":171.2552190465},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-24","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"474511.42981222225","tput_per_gpu":811.4145449789,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":226.5558181441},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"2044989.5033012282","tput_per_gpu":3496.9320506451,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":134.0967949325},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":68342.3419780292,"tput_per_gpu":93.6290085099,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":91.5325683267},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"130492.42864070175","tput_per_gpu":223.1420529756,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":228.006119413},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"2260389.7788726315","tput_per_gpu":3865.2665218722,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":109.9829819651}]