[{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"12760691.631744854","tput_per_gpu":21820.7826902837,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":24.4736869747},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1234885.4101906433","tput_per_gpu":2111.654051426,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":140.2238355375},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"20318.885900877194","tput_per_gpu":34.7452948905,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":216.8267391944},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":873053.8373279562,"tput_per_gpu":1196.0837571393,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":48.2524444438},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"222955.84744357664","tput_per_gpu":305.4495109977,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":117.5282958874},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":16,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":555807.7864120438,"tput_per_gpu":761.4566673845,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":46.3318678023},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":1077815.880521752,"tput_per_gpu":1476.6077563148,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":7.4419305674},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":7308953.617269636,"tput_per_gpu":10013.2664556594,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":9.5780493514},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"2190012.6172754746","tput_per_gpu":3000.3172856674,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":90.0229111009},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"124196.79582372264","tput_per_gpu":170.1496102785,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":143.5998097025},{"hw":"H100","tp":2,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":367945.6250881752,"tput_per_gpu":504.0855063708,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":85.6864643889},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":7589655.394160585,"dataset_name":"random","tput_per_gpu":10397.82789,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"126.0570971","input_tput_per_gpu":5198.913946,"total_input_tokens":2621440,"output_tput_per_gpu":5198.913946,"total_output_tokens":2621440,"tput_requests_per_sec":"20.3082576","p50_e2el":25.24915933,"p50_tpot":0.024208873,"p50_ttft":0.483482141,"p50_intvty":41.30716847},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":398164.2956880702,"tput_per_gpu":680.8609456266,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":185.6194930276},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":1024,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"17745122.456125207","tput_per_gpu":30344.1593999741,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":9.152319081},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":3817669.272998304,"tput_per_gpu":6528.2144568271,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":44.1448054466},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":512,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2798792.1392275915","tput_per_gpu":3834.3452307418,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":16.7031945215},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"14882.908635693431","tput_per_gpu":20.3895848309,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":186.0755805282},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":77021.12361292397,"tput_per_gpu":131.7061213781,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":195.3116814114},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":691913.0007065497,"tput_per_gpu":1183.1712312082,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":164.7650461411},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":64,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":908740.3263671533,"tput_per_gpu":1244.974247123,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":19.1234962726},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":2,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"181939.76357978102","tput_per_gpu":249.2574761043,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":111.2794521935},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"11842730.226277372","dataset_name":"random","tput_per_gpu":16224.54041,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"323.1450548","input_tput_per_gpu":8112.270205,"total_input_tokens":2621440,"output_tput_per_gpu":8112.270205,"total_output_tokens":2621440,"tput_requests_per_sec":"7.922138872","p50_e2el":63.89037131,"p50_tpot":0.061819658,"p50_ttft":0.648861384,"p50_intvty":16.17608437},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1318603.9427931388","tput_per_gpu":1806.4874016266,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":35.0400025015},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"434683.60491394164","tput_per_gpu":595.5165387321,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":134.8498766218},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"2022712.1311768615","tput_per_gpu":2771.1156197123,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":44.350265178},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1091072.7456944527","tput_per_gpu":1494.7696616014,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":96.6651069752},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":3853229.415925541,"tput_per_gpu":7321.135890258528,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":"58.899446371702524"},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":5625382.455617967,"tput_per_gpu":"10688.226665674138","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":"42.886016219933296"},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":8464149.421807243,"tput_per_gpu":"16081.883901433761","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":32.14595116596881},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":1024,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"11690090.497344982","tput_per_gpu":"22211.171944955466","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":"22.170125120948978"},{"hw":"H100","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1727017.3984578103","tput_per_gpu":2366.0138358872,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":29.0966840321},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":4099404.340875913,"dataset_name":"random","tput_per_gpu":5616.183947,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"58.34566729","input_tput_per_gpu":2808.091973,"total_input_tokens":655360,"output_tput_per_gpu":2808.091973,"total_output_tokens":655360,"tput_requests_per_sec":"10.96910927","p50_e2el":11.38600505,"p50_tpot":0.010838823,"p50_ttft":0.297889425,"p50_intvty":92.26094265},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":629618.4547445256,"dataset_name":"random","tput_per_gpu":862.577283,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"23.7428001","input_tput_per_gpu":431.2886415,"total_input_tokens":81920,"output_tput_per_gpu":431.2886415,"total_output_tokens":81920,"tput_requests_per_sec":"3.369442512","p50_e2el":4.736418284,"p50_tpot":0.004496003,"p50_ttft":0.137007163,"p50_intvty":222.4197779},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":850542.0753173723,"tput_per_gpu":1165.2426431848,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":142.6643198006},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":128,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":2594780.18101365,"tput_per_gpu":3554.8488479887,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":13.913272014},{"hw":"H200","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-06-25","num_gpus":2,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1815499.6056368614","tput_per_gpu":2487.2344597225,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":49.9618900695},{"hw":"H200","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-25","num_gpus":2,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":1643291.762522044,"tput_per_gpu":2251.3097146552,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":73.4533475827},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":84484.40501226278,"tput_per_gpu":115.7436348668,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":59.865733731},{"hw":"B300","tp":1,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-04","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":569127.7519297857,"tput_per_gpu":"1081.3427286665928","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":138.9551934958629},{"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-04","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"315972.17662698904","tput_per_gpu":600.3471355912792,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":154.6468069333619},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":715108.9715643796,"tput_per_gpu":979.6992910432,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":118.5059705797},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":1197,"date":"2026-01-28","num_gpus":26,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"7438891.6398317665","tput_per_gpu":"14133.894115680356","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":18,"input_tput_per_gpu":"18146.891321948686","output_tput_per_gpu":5104.65040157661,"p50_itl":4.311356493970379,"mean_itl":3.876006386375147,"p50_e2el":"25.350019320612773","p50_tpot":0.0235543225688392,"p50_ttft":3.791163396439515,"mean_e2el":"25.982359672289316","mean_tpot":0.02313682220567831,"mean_ttft":4.679867667079543,"p50_intvty":42.45505244642159,"mean_intvty":"43.221147273828166"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":271,"date":"2026-01-28","num_gpus":18,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1450622.0190386267","tput_per_gpu":2756.181836173391,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":12393.6815508794,"output_tput_per_gpu":"1551.4943718351394","p50_itl":2.307543207425624,"mean_itl":"2.0389398121141986","p50_e2el":9.553317219484597,"p50_tpot":0.009782980208412697,"p50_ttft":0.5528178260428831,"mean_e2el":9.42914099604857,"mean_tpot":0.009358542919091674,"mean_ttft":0.8114229454576447,"p50_intvty":102.2183402906272,"mean_intvty":"106.85424094812598"},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":2596000.678832117,"dataset_name":"random","tput_per_gpu":3556.52093,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"46.90361814","input_tput_per_gpu":1746.560356,"total_input_tokens":327680,"output_tput_per_gpu":1746.560356,"total_output_tokens":327680,"tput_requests_per_sec":"6.822501391","p50_e2el":8.250320349999999,"p50_tpot":0.007781648,"p50_ttft":0.289694052,"p50_intvty":117.0478049},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"16892674.175438598","tput_per_gpu":28886.47284,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"81.9714","input_tput_per_gpu":14479.36,"kv_cache_precision":"FP8","total_input_tokens":2373787,"output_tput_per_gpu":14407.11,"total_output_tokens":2361942,"tput_requests_per_sec":"31.23041","p50_e2el":8.583983,"p50_tpot":0.008317,"p50_ttft":0.075616,"p50_intvty":120.2346},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":384,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":19579697.57309942,"tput_per_gpu":33481.28285,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"106.0759","input_tput_per_gpu":16795.64,"kv_cache_precision":"FP8","total_input_tokens":3563226,"output_tput_per_gpu":16685.64,"total_output_tokens":3539890,"tput_requests_per_sec":"36.20049","p50_e2el":11.18067,"p50_tpot":0.010841,"p50_ttft":0.089843,"p50_intvty":92.23836},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"4035826.7061335766","tput_per_gpu":5529.082587403,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":43.5565945642},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":7759056.902640074,"tput_per_gpu":10629.9079566169,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":5.30107158},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":256,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":3199072.323837331,"tput_per_gpu":"5470.4136737618355","model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":"100.26314439618966"},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-24","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"474511.42981225066","tput_per_gpu":811.4145449789486,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":226.5558181440583},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":2044989.503301222,"tput_per_gpu":"3496.9320506450895","model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":"134.09679493254953"},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"109039.76233576643","dataset_name":"random","tput_per_gpu":149.3844744,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"15.37894088","input_tput_per_gpu":74.54186922,"total_input_tokens":9171,"output_tput_per_gpu":74.84260515,"total_output_tokens":9208,"tput_requests_per_sec":"0.650239836","p50_e2el":3.341955062,"p50_tpot":0.003241737,"p50_ttft":0.025658196,"p50_intvty":308.4766055},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"223022.09073474456","tput_per_gpu":305.5402643066,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":147.5860442873},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-25","num_gpus":4,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":2342436.214838759,"tput_per_gpu":3209.1376143291,"model_display":"MiniMax-M3","spec_decoding":"off","date_timestamp":1782370800,"serving_config":"agg","p50_intvty":24.890937309},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":161109.6528130609,"tput_per_gpu":"275.49750631033413","model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":"318.52317827760055"},{"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-04","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"2579645.1940707397","tput_per_gpu":4901.325868734405,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":79.05701770711066},{"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-04","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":94359.39031459269,"tput_per_gpu":179.2828415977261,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":"186.80574778139427"},{"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-04","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"174425.01486816205","tput_per_gpu":331.4075282495079,"model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":"171.16725978712222"},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"50906.741010218975","tput_per_gpu":69.742235184,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":72.634450126},{"hw":"B300","tp":1,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-04","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":923313.0848602148,"tput_per_gpu":"1754.2948612344082","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":113.0023458724614},{"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-04","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1581800.8336760355","tput_per_gpu":"3005.4215839844674","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":97.27253342836282},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":52,"date":"2026-01-28","num_gpus":18,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1178205.6460119977","tput_per_gpu":"2238.5907274227957","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":17900.38469602766,"output_tput_per_gpu":"280.86648134718723","p50_itl":1.753233201103285,"mean_itl":1.520754710946752,"p50_e2el":9.998001151019707,"p50_tpot":0.007056075616404063,"p50_ttft":4.046001616166905,"mean_e2el":"10.103149035872551","mean_tpot":0.006540426738116921,"mean_ttft":4.058149433229119,"p50_intvty":141.7218372313338,"mean_intvty":"152.89522229063508"},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":512,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":4649408.07335247,"tput_per_gpu":7950.487805432723,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":67.68769255025258},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-24","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":774867.4462341134,"tput_per_gpu":1325.023333060334,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":"179.94862494498005"},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":5595799.838952709,"tput_per_gpu":9568.817724609133,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":40.9644202385574},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"124798.62561729927","tput_per_gpu":170.9741170957,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":43.24832588},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"240134.92540109492","tput_per_gpu":328.9848477995,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":22.2767288233},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":32,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"175738.16320846716","tput_per_gpu":240.7612835956,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":30.7241214251},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-24","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"1193925.0238224994","tput_per_gpu":2041.611790736474,"model_display":"Qwen 3.5","spec_decoding":"mtp","date_timestamp":1782284400,"serving_config":"agg","p50_intvty":162.2807013289114},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1029105.5587245986","tput_per_gpu":1409.8746154527,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":90.0582596635},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":28757.24417109489,"tput_per_gpu":39.3974245144,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":81.3907186517},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":25,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"135259.00179314555","tput_per_gpu":256.9921034069765,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"2703.2809379654905","output_tput_per_gpu":134.6776616790508,"p50_itl":1.037669722456485,"mean_itl":0.9267604010714091,"p50_e2el":4.188428748515435,"p50_tpot":0.004442142441192792,"p50_ttft":"0.12816249742172658","mean_e2el":4.112898063314147,"mean_tpot":0.004268328720238588,"mean_ttft":0.1982623645113781,"p50_intvty":"225.11659930731142","mean_intvty":"234.28373622172728"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":60,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"252643.33433578289","tput_per_gpu":480.0223352379875,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"5032.2650897641515","output_tput_per_gpu":"252.41019751167929","p50_itl":"1.3597503451164812","mean_itl":1.20893069184662,"p50_e2el":5.480907928431407,"p50_tpot":0.005769511478436915,"p50_ttft":0.1678955900715664,"mean_e2el":5.288806735235655,"mean_tpot":0.005477927089800219,"mean_ttft":"0.23749159428485048","p50_intvty":"173.32489999151915","mean_intvty":"182.55080500468475"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":10,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":67461.27730265421,"tput_per_gpu":128.176426875043,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"1351.5168321981364","output_tput_per_gpu":67.00940660888834,"p50_itl":0.8281047309283167,"mean_itl":0.7350380142327166,"p50_e2el":"3.3680827184580266","p50_tpot":"0.0034741356025225346","p50_ttft":0.1181410294957459,"mean_e2el":3.251156003009528,"mean_tpot":0.003361764081598057,"mean_ttft":"0.17134704606607556","p50_intvty":287.8413839902824,"mean_intvty":"297.46287238711807"},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":8,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":118893.6941520468,"tput_per_gpu":203.308217,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"11.29996","input_tput_per_gpu":101.4495,"kv_cache_precision":"FP8","total_input_tokens":9171,"output_tput_per_gpu":101.8588,"total_output_tokens":9208,"tput_requests_per_sec":"0.884959","p50_e2el":1.042741,"p50_tpot":0.000983,"p50_ttft":0.037488,"p50_intvty":1017.655},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"254723.40192982458","tput_per_gpu":435.5770173,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"10.54865","input_tput_per_gpu":217.3501,"kv_cache_precision":"FP8","total_input_tokens":9171,"output_tput_per_gpu":218.227,"total_output_tokens":9208,"tput_requests_per_sec":"0.947989","p50_e2el":1.123263,"p50_tpot":0.00106,"p50_ttft":0.038532,"p50_intvty":943.0912},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"488482.31295620446","tput_per_gpu":669.22076875,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":27.9659859892},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":68342.3419780292,"tput_per_gpu":93.6290085099,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":91.5325683267},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2900495.9751145258","tput_per_gpu":3973.6794859069,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":30.3352731133},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":15999.78503832117,"tput_per_gpu":21.9197055025,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":91.8326038197},{"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":32,"date":"2026-07-14","num_gpus":8,"sequence":"1K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"175527.12739160584","tput_per_gpu":240.4721645265,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":31.0766654313},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":2124084.11871345,"tput_per_gpu":3632.183843,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"20.30837","input_tput_per_gpu":1830.747,"kv_cache_precision":"FP8","total_input_tokens":148718,"output_tput_per_gpu":1801.437,"total_output_tokens":146337,"tput_requests_per_sec":"7.878523","p50_e2el":2.134473,"p50_tpot":0.002048,"p50_ttft":0.039658,"p50_intvty":488.3487},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"2336776.0953216376","tput_per_gpu":3995.887123,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"23.14442","input_tput_per_gpu":2008.194,"kv_cache_precision":"FP8","total_input_tokens":92957,"output_tput_per_gpu":1987.693,"total_output_tokens":92008,"tput_requests_per_sec":"4.320695","p50_e2el":2.36565,"p50_tpot":0.002274,"p50_ttft":0.039446,"p50_intvty":439.7722},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"1275137.1608187135","tput_per_gpu":2180.484545,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"17.01858","input_tput_per_gpu":1091.704,"kv_cache_precision":"FP8","total_input_tokens":74317,"output_tput_per_gpu":1088.781,"total_output_tokens":74118,"tput_requests_per_sec":"4.700745","p50_e2el":1.750312,"p50_tpot":0.001674,"p50_ttft":0.037754,"p50_intvty":597.3522},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":826534.7093567251,"tput_per_gpu":1413.374353,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"13.00363","input_tput_per_gpu":705.2645,"kv_cache_precision":"FP8","total_input_tokens":9171,"output_tput_per_gpu":708.1099,"total_output_tokens":9208,"tput_requests_per_sec":"0.769016","p50_e2el":1.196685,"p50_tpot":0.001142,"p50_ttft":0.028784,"p50_intvty":875.9301},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1534964.1624744528","tput_per_gpu":2102.90090259,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":137.5382215279},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":73059.41003846716,"tput_per_gpu":100.0913917527,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":300.9730757744},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":283932.6677031387,"tput_per_gpu":388.9877547533,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":211.701473006},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1026654.0812368614","tput_per_gpu":1406.5160912945,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":90.7973850019},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":311448.3213751095,"tput_per_gpu":426.6842002839,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":50.3579594834},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2506588.6238496355","tput_per_gpu":3434.026414674,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":53.2946584656},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":160,"date":"2026-03-23","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":824974.5135229748,"tput_per_gpu":"1130.2150835264754","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":2679.962484529921,"output_tput_per_gpu":"200.36664292440827","p50_itl":"3.9621351920068264","mean_itl":"3.4324930780430374","p50_e2el":"17.796283358940855","p50_tpot":"0.016198696561501504","p50_ttft":3.001771781477146,"mean_e2el":"17.620267784017777","mean_tpot":"0.015171156326895998","mean_ttft":3.677910163874767,"p50_intvty":61.73336207659088,"mean_intvty":65.91455380544477},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":32,"date":"2026-01-28","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":524194.4929532111,"tput_per_gpu":995.9695366111011,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"15038.300102143276","output_tput_per_gpu":118.3238762653402,"p50_itl":"1.0309762844117358","mean_itl":0.8967610940179305,"p50_e2el":7.475854144548066,"p50_tpot":0.004220500760906727,"p50_ttft":3.899220559047535,"mean_e2el":7.399177072955354,"mean_tpot":0.003899523362829599,"mean_ttft":"3.7953188362582297","p50_intvty":"236.93870861550596","mean_intvty":"256.44159733264763"},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":28,"date":"2026-03-23","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":272436.6509276795,"tput_per_gpu":373.2382117709209,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"2653.8234255887655","output_tput_per_gpu":47.44032408265734,"p50_itl":"2.0654048461001366","mean_itl":1.779713051492671,"p50_e2el":9.385275139007717,"p50_tpot":0.008543635024882873,"p50_ttft":1.559641165076755,"mean_e2el":9.303517086410595,"mean_tpot":0.008049056510393579,"mean_ttft":"1.8922836605484399","p50_intvty":"117.04619837897503","mean_intvty":"124.23816365417747"},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":2198,"date":"2026-01-28","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8351617.862749809,"tput_per_gpu":"15868.073939224638","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":20,"input_tput_per_gpu":"19744.787628126483","output_tput_per_gpu":6176.289716970031,"p50_itl":"7.1040376119781286","mean_itl":6.281262535290535,"p50_e2el":38.52322070696391,"p50_tpot":0.03848032634036372,"p50_ttft":"2.7733500495087355","mean_e2el":"39.720939040408346","mean_tpot":0.03747442112920176,"mean_ttft":5.193142044788798,"p50_intvty":25.98730559493867,"mean_intvty":"26.684868501431097"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":120,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":397226.9582514457,"tput_per_gpu":754.7312206777469,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":7939.809450794024,"output_tput_per_gpu":"395.47730917193314","p50_itl":"1.7068251119926572","mean_itl":1.504208827984031,"p50_e2el":6.863900861470029,"p50_tpot":0.007236778210798999,"p50_ttft":0.1890602840576321,"mean_e2el":"6.6649840745679105","mean_tpot":0.006917935158810374,"mean_ttft":0.3159695549470295,"p50_intvty":138.1830382072179,"mean_intvty":144.551802964855},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":20,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"116353.03737904492","tput_per_gpu":"221.07077102018536","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"2322.6602891001153","output_tput_per_gpu":"115.99129511618887","p50_itl":0.9759253894444555,"mean_itl":0.8706814760590146,"p50_e2el":"3.9051218144595623","p50_tpot":0.00414243905712611,"p50_ttft":"0.11817648296710104","mean_e2el":"3.8018023827834986","mean_tpot":0.003950340060042362,"mean_ttft":0.1710609053086955,"p50_intvty":241.4036721384545,"mean_intvty":"253.14276361040078"},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"1710434.3416058396","dataset_name":"random","tput_per_gpu":2343.295048,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"34.95931939","input_tput_per_gpu":1171.647524,"total_input_tokens":327680,"output_tput_per_gpu":1171.647524,"total_output_tokens":327680,"tput_requests_per_sec":"9.15349628","p50_e2el":6.970084015,"p50_tpot":0.006520528,"p50_ttft":0.299583494,"p50_intvty":153.361805},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":201444.7283941606,"dataset_name":"random","tput_per_gpu":275.9792779,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"18.55211753","input_tput_per_gpu":137.9896389,"total_input_tokens":10240,"output_tput_per_gpu":137.9896389,"total_output_tokens":10240,"tput_requests_per_sec":"0.539022027","p50_e2el":3.709422564,"p50_tpot":0.003589207,"p50_ttft":0.037663763,"p50_intvty":278.6130722},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"205050.48252591243","tput_per_gpu":280.9191610605,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":66.8374425706},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"2002916.6404117518","tput_per_gpu":2743.9957973641,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":85.0708523227},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":1229,"date":"2026-02-03","num_gpus":38,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":2816161.067549051,"tput_per_gpu":5970.261463203988,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":18902.86268335913,"output_tput_per_gpu":3545.398734424898,"p50_itl":2.018231824535178,"mean_itl":1.784913617429043,"p50_e2el":9.431017198512563,"p50_tpot":0.008557226086643908,"p50_ttft":"1.5571892604930326","mean_e2el":9.413059297616146,"mean_tpot":0.008201435622821959,"mean_ttft":1.864719287748728,"p50_intvty":"116.86029910566427","mean_intvty":"121.92987252345448"},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":48,"date":"2026-03-23","num_gpus":56,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":324737.0684863247,"tput_per_gpu":"444.88978382626476","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":48,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":2766.853336069788,"output_tput_per_gpu":"57.895858452344264","p50_itl":"2.1325586325256154","mean_itl":"1.9446091976934263","p50_e2el":"15.565871494007297","p50_tpot":0.009120842769664744,"p50_ttft":7.121810148470104,"mean_e2el":"15.294538287106842","mean_tpot":0.008704023928945285,"mean_ttft":7.240615400263535,"p50_intvty":"109.63899118247349","mean_intvty":"114.88939002964985"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":200,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"501823.77163829014","tput_per_gpu":953.4651661127513,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"10018.485021454651","output_tput_per_gpu":500.2141733456562,"p50_itl":"1.8583798034815118","mean_itl":"1.6333709418746276","p50_e2el":8.869902378530242,"p50_tpot":0.00785421199896822,"p50_ttft":"1.7260119415586814","mean_e2el":8.782491409745534,"mean_tpot":0.00751560261297044,"mean_ttft":"1.8782327033884358","p50_intvty":"127.32022004643704","mean_intvty":133.0565293958196},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-03-23","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":1152917.830864599,"tput_per_gpu":"1579.4974282845005","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":3948.62728782155,"output_tput_per_gpu":987.2149634002382,"p50_itl":5.779115913901478,"mean_itl":5.116026180101083,"p50_e2el":27.80117062793579,"p50_tpot":"0.024539258218348094","p50_ttft":5.098565860535018,"mean_e2el":28.54202498249117,"mean_tpot":"0.023912256876124593","mean_ttft":6.529623042377852,"p50_intvty":"40.751028050729595","mean_intvty":41.81955744204383},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":11,"date":"2026-01-28","num_gpus":42,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":39475.85164543328,"tput_per_gpu":75.00411812632323,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":789.0391299167156,"output_tput_per_gpu":"39.302367536803615","p50_itl":0.7440155820222571,"mean_itl":0.6618175429718222,"p50_e2el":6.586459893966094,"p50_tpot":"0.0031857889096050243","p50_ttft":3.685699826106429,"mean_e2el":6.123232413671741,"mean_tpot":0.003086418501081695,"mean_ttft":"3.2834338487274097","p50_intvty":313.8939924691936,"mean_intvty":324.0001314305013},{"dp":1,"hw":"H200","pp":1,"tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":9456430.175182482,"dataset_name":"random","tput_per_gpu":12955.30934,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"202.3448403","input_tput_per_gpu":6477.654671,"total_input_tokens":2621440,"output_tput_per_gpu":6477.654671,"total_output_tokens":2621440,"tput_requests_per_sec":"12.65166928","p50_e2el":39.91009949,"p50_tpot":0.038500541,"p50_ttft":0.524046413,"p50_intvty":25.97366124},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":2988232.65499511,"tput_per_gpu":4093.8787373433,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":65.3300512735},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":283277.5699298541,"tput_per_gpu":388.0902708039,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":90.5563295506},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":32,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":720523.4570332847,"tput_per_gpu":987.1171361356,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":30.4722272525},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":408822.8859940146,"tput_per_gpu":560.0873538118,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":69.4047299976},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":333,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":1067671.001768995,"tput_per_gpu":"2263.4625237502696","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":19225.52891934771,"output_tput_per_gpu":"1203.3333740254295","p50_itl":"1.4978534699766897","mean_itl":1.323840396575669,"p50_e2el":7.534498858498409,"p50_tpot":0.006355386524548465,"p50_ttft":"1.6955248714948539","mean_e2el":7.48946537534434,"mean_tpot":0.006073885930893127,"mean_ttft":"1.8947490083770204","p50_intvty":"157.34684210588557","mean_intvty":"164.63924600786112"},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":2253,"date":"2026-02-03","num_gpus":22,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":5659108.919853944,"tput_per_gpu":"11997.310910090362","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":"21986.622964483835","output_tput_per_gpu":8251.31888969281,"p50_itl":"2.5215269179898314","mean_itl":"2.2966773211463547","p50_e2el":"14.483616410027025","p50_tpot":"0.014206319296476084","p50_ttft":1.401400372007629,"mean_e2el":"14.939291313216753","mean_tpot":"0.014023291159628358","mean_ttft":2.019691479533487,"p50_intvty":70.39120965330216,"mean_intvty":71.30993634924299},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":12,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":81307.3395783962,"tput_per_gpu":"172.37155990619993","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"1472.7323262995267","output_tput_per_gpu":91.099012006617,"p50_itl":0.8782400769996457,"mean_itl":0.7781435256087755,"p50_e2el":"3.7379440465010703","p50_tpot":"0.0037285514639768765","p50_ttft":0.3578497510170564,"mean_e2el":"3.6398164080198816","mean_tpot":0.003572348575124443,"mean_ttft":0.3716137303301366,"p50_intvty":"268.20066979400065","mean_intvty":"279.92789028578073"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":5,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"33725.733002716115","tput_per_gpu":71.49855396575816,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":608.9784318577393,"output_tput_per_gpu":37.90606159750934,"p50_itl":0.7575018520001322,"mean_itl":0.6646102177452967,"p50_e2el":3.786653770977864,"p50_tpot":0.003244997297561225,"p50_ttft":0.5377196299959905,"mean_e2el":3.738447680550162,"mean_tpot":0.00309706108928404,"mean_ttft":0.8806239112839103,"p50_intvty":308.1666665027885,"mean_intvty":322.8867533352963},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":12926,"date":"2026-01-28","num_gpus":26,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":7671806.953324033,"tput_per_gpu":"14576.433211315663","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":10,"input_tput_per_gpu":"18950.892090478046","output_tput_per_gpu":"11842.396411839174","p50_itl":"10.640068282024004","mean_itl":9.525670334418315,"p50_e2el":"58.451632245094515","p50_tpot":0.05994752577887179,"p50_ttft":"3.4536425890401006","mean_e2el":60.77319273568867,"mean_tpot":0.0581497361195894,"mean_ttft":7.259501028840867,"p50_intvty":16.68125559824931,"mean_intvty":"17.196982595818202"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":8609,"date":"2026-01-28","num_gpus":26,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":7257145.211366724,"tput_per_gpu":"13788.575901596776","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":10,"input_tput_per_gpu":17928.27118177515,"output_tput_per_gpu":11201.26635148529,"p50_itl":7.524473516037688,"mean_itl":"6.7733019084734085","p50_e2el":"41.030647561419755","p50_tpot":0.04232217676076305,"p50_ttft":"2.1228908135090023","mean_e2el":42.68912139114758,"mean_tpot":0.04137596942168306,"mean_ttft":4.613786584753449,"p50_intvty":"23.628274265115337","mean_intvty":24.16861801613645},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":9,"date":"2026-03-23","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"112596.14075579442","tput_per_gpu":"154.25671283543835","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"1096.0663917866113","output_tput_per_gpu":"19.712472985270768","p50_itl":"1.4488349300809205","mean_itl":"1.2763731882396643","p50_e2el":6.588520267978311,"p50_tpot":0.00598152700820218,"p50_ttft":1.334036235930398,"mean_e2el":7.034097404428758,"mean_tpot":0.005573140235228842,"mean_ttft":"1.9144230640493334","p50_intvty":"167.18139007459936","mean_intvty":"179.43205406510614"},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":896,"date":"2026-03-23","num_gpus":64,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":721525.2967092205,"tput_per_gpu":988.4896564916321,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":3953.3121321026,"output_tput_per_gpu":564.9435885472082,"p50_itl":5.13295093504712,"mean_itl":"4.5345815648242045","p50_e2el":24.07485940703191,"p50_tpot":"0.021792099770738503","p50_ttft":4.011317240307108,"mean_e2el":"24.846131647857327","mean_tpot":0.02119890996472177,"mean_ttft":5.335790986408912,"p50_intvty":"45.888189321836585","mean_intvty":47.17223676425594},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-03-23","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1690299.8447403056","tput_per_gpu":"2315.7107872942183","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"3473.4638911101815","output_tput_per_gpu":"1736.8342353862365","p50_itl":5.612508881022222,"mean_itl":5.226481198863922,"p50_e2el":31.62627227907069,"p50_tpot":0.03193160360476903,"p50_ttft":"1.8319238550029695","mean_e2el":"32.597232527006625","mean_tpot":"0.031994770350886104","mean_ttft":"3.1390697234239724","p50_intvty":"31.316936423783257","mean_intvty":"31.255107913981472"},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-01-28","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"180078.20733577266","tput_per_gpu":"342.14859393796803","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":5162.729843656699,"output_tput_per_gpu":40.86226583054734,"p50_itl":0.7757313795154914,"mean_itl":0.6886392038409562,"p50_e2el":5.6710503725335,"p50_tpot":0.003138802230974757,"p50_ttft":2.976597201428376,"mean_e2el":5.47180453153851,"mean_tpot":"0.0029003202073141577","mean_ttft":2.7861119946494,"p50_intvty":"318.59286645449123","mean_intvty":344.789515819031},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":8192,"osl":1024,"conc":308,"date":"2026-02-03","num_gpus":48,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"3291308.7043642006","tput_per_gpu":6977.574453252105,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":"18606.061166702606","output_tput_per_gpu":"1163.3310965268536","p50_itl":"1.6312592354952358","mean_itl":"1.4225727403372335","p50_e2el":7.300424157991074,"p50_tpot":0.00665646398306822,"p50_ttft":"1.1916593965142965","mean_e2el":"7.1409414125764785","mean_tpot":0.006174597921174912,"mean_ttft":1.456193101125699,"p50_intvty":150.2299122392399,"mean_intvty":"161.95386529876563"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":8192,"osl":1024,"conc":180,"date":"2026-02-03","num_gpus":40,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"2231514.7829074776","tput_per_gpu":4730.811339763853,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"21030.893481571457","output_tput_per_gpu":655.7908043119518,"p50_itl":"1.4644192999985535","mean_itl":1.277520483178387,"p50_e2el":7.503871290507959,"p50_tpot":0.005984274275085108,"p50_ttft":"2.1143423394823913","mean_e2el":"7.3823316377426895","mean_tpot":"0.0055251656252957825","mean_ttft":2.302058639433429,"p50_intvty":"167.10464026747474","mean_intvty":180.9900494967454},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":3226,"date":"2026-02-03","num_gpus":6,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8357326.118558963,"tput_per_gpu":17717.531371345,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"26581.833572570853","output_tput_per_gpu":"13285.380270732076","p50_itl":9.007933673507068,"mean_itl":7.829054975546211,"p50_e2el":53.56805461450131,"p50_tpot":0.04987195362427589,"p50_ttft":5.989211072010221,"mean_e2el":54.2908676289247,"mean_tpot":"0.047827870251332734","mean_ttft":"10.276338087891613","p50_intvty":"20.051350054055945","mean_intvty":"20.908311299354477"},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":1127,"date":"2026-02-03","num_gpus":42,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":6863618.555416359,"tput_per_gpu":14550.87133748268,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":26,"input_tput_per_gpu":20893.15410327928,"output_tput_per_gpu":4244.661843063211,"p50_itl":3.622992394026369,"mean_itl":3.121206131533661,"p50_e2el":"14.804202232480748","p50_tpot":0.01455758278422863,"p50_ttft":"1.3658870729559567","mean_e2el":"14.553257917177158","mean_tpot":"0.013527406277272691","mean_ttft":2.101390356177955,"p50_intvty":68.69272288002226,"mean_intvty":73.92400135716288},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":5,"date":"2026-02-03","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"148704.27171974472","tput_per_gpu":315.2530560458588,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":4758.161690033561,"output_tput_per_gpu":37.57126642162738,"p50_itl":0.789531749964226,"mean_itl":0.6970344406914022,"p50_e2el":"3.7631857955129817","p50_tpot":0.003201082352515862,"p50_ttft":0.8078803464886732,"mean_e2el":3.691280626163352,"mean_tpot":0.003039379524793688,"mean_ttft":0.8892023601802066,"p50_intvty":312.3943372509798,"mean_intvty":329.0145214977322},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":48,"date":"2026-03-23","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":560918.9472044705,"tput_per_gpu":768.4589576701245,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"2730.9662597789575","output_tput_per_gpu":"114.28985696718011","p50_itl":2.870605044066906,"mean_itl":"2.5067171161494657","p50_e2el":"15.598151015932672","p50_tpot":"0.012863503543652132","p50_ttft":"3.7256986449938267","mean_e2el":"15.659430478020417","mean_tpot":"0.012356145748898595","mean_ttft":"4.2225340591792095","p50_intvty":77.73931857728441,"mean_intvty":80.93138591288779},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":512,"date":"2026-03-23","num_gpus":72,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"471302.08435041475","tput_per_gpu":645.6838555600682,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":64,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"2904.2384548478794","output_tput_per_gpu":"363.36453064909176","p50_itl":4.2018386045238,"mean_itl":"3.9501280332956017","p50_e2el":"18.861431284924038","p50_tpot":0.01835804596147137,"p50_ttft":"1.3376573454588652","mean_e2el":"19.239580527059708","mean_tpot":"0.018438245967498552","mean_ttft":"2.2648312946147144","p50_intvty":54.47202834652079,"mean_intvty":54.23509382414786},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":2342,"date":"2026-01-28","num_gpus":12,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":6658188.810011935,"tput_per_gpu":"12650.558739022677","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":18971.45868281698,"output_tput_per_gpu":9490.108767125525,"p50_itl":4.488675082917325,"mean_itl":4.065855791068461,"p50_e2el":"26.584189099492505","p50_tpot":"0.025266686877534875","p50_ttft":3.406632080557756,"mean_e2el":27.3197190430005,"mean_tpot":0.02483820854159049,"mean_ttft":4.440166145387608,"p50_intvty":39.57780475322708,"mean_intvty":40.26055254047585},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":181,"date":"2026-01-28","num_gpus":14,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":4140818.373782102,"tput_per_gpu":7867.554910185994,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":"16321.394307722447","output_tput_per_gpu":"1527.1753620336551","p50_itl":2.56500584888272,"mean_itl":"2.2340110614118753","p50_e2el":"13.253701606416143","p50_tpot":"0.010508398953154544","p50_ttft":3.752112309448421,"mean_e2el":12.91247708422173,"mean_tpot":0.009723195936432059,"mean_ttft":3.970261836479912,"p50_intvty":95.16197514558651,"mean_intvty":102.8468423898646},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":654,"date":"2026-01-28","num_gpus":10,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":3844119.452105054,"tput_per_gpu":"7303.8269589996025","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1769587200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"18264.315941759003","output_tput_per_gpu":4563.704713309752,"p50_itl":3.215434278943576,"mean_itl":2.825896552530392,"p50_e2el":"15.862855603103526","p50_tpot":0.01358851711919394,"p50_ttft":"3.3765829774783924","mean_e2el":15.71562655507982,"mean_tpot":"0.012986853768726554","mean_ttft":3.768220123604605,"p50_intvty":73.59154727689074,"mean_intvty":77.00094401679372},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"41531.700733737714","tput_per_gpu":78.91023139410166,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":39.33700589661879,"output_tput_per_gpu":"39.573225497482866","p50_itl":1.312467972500599,"p90_itl":"1.3313824329088675","mean_itl":1.177495770772194,"p50_e2el":5.901352243992733,"p50_tpot":"0.0057952412948427175","p50_ttft":"0.11723010698915459","p90_e2el":7.486617347775609,"p90_tpot":0.006576019012236227,"p90_ttft":"1.6904430321912505","mean_e2el":5.820420388691128,"mean_tpot":0.005614867298463764,"mean_ttft":0.6394392232963583,"p50_intvty":"172.55536898695087","p90_intvty":"152.06768686940615","mean_intvty":"178.09859910199506"},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":666,"date":"2026-02-03","num_gpus":36,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":5745264.126273823,"tput_per_gpu":"12179.959947700505","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":20,"input_tput_per_gpu":"19489.467203789223","output_tput_per_gpu":3043.075877589611,"p50_itl":2.544716412987327,"mean_itl":2.20566259319054,"p50_e2el":"12.142953218019102","p50_tpot":"0.010320322778277746","p50_ttft":2.721223544504028,"mean_e2el":"11.894429282512467","mean_tpot":0.009591796674166915,"mean_ttft":"3.0691295953088438","p50_intvty":96.89619418733722,"mean_intvty":"104.25575457549552"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":12,"date":"2026-02-03","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"342794.63497908966","tput_per_gpu":726.7246261556701,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":10986.21171809185,"output_tput_per_gpu":85.50668290965878,"p50_itl":0.9008071229909547,"mean_itl":0.8114574016380014,"p50_e2el":3.941425225493731,"p50_tpot":"0.0038037855643335288","p50_ttft":0.5019430844986346,"mean_e2el":"3.8309279630329303","mean_tpot":"0.0035232093452133824","mean_ttft":0.6121470498692361,"p50_intvty":"262.89599744438084","mean_intvty":283.8321263420228},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":24,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":132901.1209076174,"tput_per_gpu":"281.75037632414893","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"2400.1872437431207","output_tput_per_gpu":"149.34807211046322","p50_itl":"1.0697207389748655","mean_itl":0.9500149552184466,"p50_e2el":4.503277394978795,"p50_tpot":0.004566908470701551,"p50_ttft":"0.38739624302252196","mean_e2el":4.398056330384376,"mean_tpot":0.004355970472100667,"mean_ttft":0.4040352574835803,"p50_intvty":"218.96650795946076","mean_intvty":229.5699675663205},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":24,"date":"2026-02-03","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"471076.92697124684","tput_per_gpu":998.6830851790432,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"15090.997197889032","output_tput_per_gpu":"117.91345313466894","p50_itl":"1.0178682089899667","mean_itl":0.8924206767120818,"p50_e2el":5.452911655505886,"p50_tpot":0.004221237027603705,"p50_ttft":"1.9026468619995285","mean_e2el":5.573646852141489,"mean_tpot":0.003926335198725436,"mean_ttft":"1.9704985239167097","p50_intvty":"236.89738184820106","mean_intvty":"254.69042997771032"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":48,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":179785.6728078375,"tput_per_gpu":"381.14562635261547","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":3228.829345046255,"output_tput_per_gpu":"203.16539393426297","p50_itl":"1.1304270030232146","mean_itl":"1.0034410328311425","p50_e2el":6.284766654513078,"p50_tpot":0.004771396156772042,"p50_ttft":"2.0564980504859705","mean_e2el":6.349060258750372,"mean_tpot":0.004558543415794125,"mean_ttft":"2.1325175706399024","p50_intvty":209.5822621185416,"mean_intvty":219.368317637443},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":2253,"date":"2026-02-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8552620.647857748,"tput_per_gpu":"18131.555773458425","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":20,"input_tput_per_gpu":"22561.532455216246","output_tput_per_gpu":7056.61406906387,"p50_itl":6.232570104999468,"mean_itl":"5.4753600121992685","p50_e2el":34.81048325699521,"p50_tpot":0.03356100164796111,"p50_ttft":"3.3582652070035692","mean_e2el":35.64663746862041,"mean_tpot":0.03270313518283074,"mean_ttft":5.517332980563301,"p50_intvty":"29.796488510370544","mean_intvty":"30.578108013478893"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":33,"date":"2026-02-03","num_gpus":26,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":760598.0670237849,"tput_per_gpu":"1612.4679020904239","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"18618.173698410643","output_tput_per_gpu":195.3257523970723,"p50_itl":1.231677994976053,"mean_itl":"1.0714976970517822","p50_e2el":6.237760724470718,"p50_tpot":0.005076128667209518,"p50_ttft":1.677924030489521,"mean_e2el":6.174575005678079,"mean_tpot":0.004690036469905913,"mean_ttft":"1.8366329692254746","p50_intvty":"197.00052255565194","mean_intvty":"213.21795820066643"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-02-03","num_gpus":34,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":57577.6463807555,"tput_per_gpu":"122.06461032720165","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1770105600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"1033.7031177887636","output_tput_per_gpu":65.08720361085402,"p50_itl":0.8192278609494679,"mean_itl":0.7204120387978249,"p50_e2el":"3.5515489155077375","p50_tpot":"0.0035179187085839043","p50_ttft":"0.28977185950498097","mean_e2el":3.431067697285471,"mean_tpot":0.003329587959960622,"mean_ttft":"0.34230123414017727","p50_intvty":"284.25898459789533","mean_intvty":300.337462780778},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":352,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"294165.86987650377","tput_per_gpu":"403.00724173081016","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"2416.3864516295557","output_tput_per_gpu":"219.97276810365148","p50_itl":"3.7587406649254262","mean_itl":"3.3297293832575194","p50_e2el":16.02494055684656,"p50_tpot":0.01597461209303012,"p50_ttft":"1.2521666130051017","mean_e2el":"16.101884752360505","mean_tpot":"0.015568193962182748","mean_ttft":"1.7717513356322094","p50_intvty":"62.599329121507104","mean_intvty":64.23352653680546},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"290009.47318645776","tput_per_gpu":551.0179990542697,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":"276.89854171607885","output_tput_per_gpu":"274.11945733819084","p50_itl":"3.0831930610584095","p90_itl":"3.3579006591811775","mean_itl":"2.8870980598676472","p50_e2el":"12.941892520990223","p50_tpot":0.01430356033195214,"p50_ttft":"0.16664908698294312","p90_e2el":"15.411388897232245","p90_tpot":"0.016053414434686428","p90_ttft":0.4035498623969009,"mean_e2el":"12.985334434996911","mean_tpot":0.013895969136964,"mean_ttft":"0.26405881243845214","p50_intvty":69.91266347624939,"p90_intvty":62.29204410491712,"mean_intvty":71.96331469533479},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":440378.9315434717,"tput_per_gpu":836.7199699325962,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":417.7080579584406,"output_tput_per_gpu":419.0119119741556,"p50_itl":4.181991025048774,"p90_itl":4.526558781974018,"mean_itl":"3.7344296226857545","p50_e2el":17.49317130097188,"p50_tpot":"0.018201672953459664","p50_ttft":"0.18425112147815526","p90_e2el":"20.772122321336067","p90_tpot":"0.021404050334262306","p90_ttft":"1.8216523942305451","mean_e2el":"17.143995082245237","mean_tpot":0.01784067201206105,"mean_ttft":0.6891647761578497,"p50_intvty":"54.940004831255145","p90_intvty":46.72012933922421,"mean_intvty":56.0517002568041},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":68301.16080872715,"tput_per_gpu":"129.77220553658157","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":64.3227581530139,"output_tput_per_gpu":65.44944738356767,"p50_itl":1.494180807028897,"p90_itl":"1.5961899073095993","mean_itl":"1.3736196521621542","p50_e2el":6.81866548594553,"p50_tpot":0.006833024525770923,"p50_ttft":"0.14026347204344347","p90_e2el":8.54149876383599,"p90_tpot":0.007647480507663373,"p90_ttft":"1.7545537921367254","mean_e2el":6.946265591244446,"mean_tpot":0.006745803162930397,"mean_ttft":0.6276153707469347,"p50_intvty":"146.34807708189467","p90_intvty":130.7620201186419,"mean_intvty":"148.24031710489413"},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":190757.1469567012,"tput_per_gpu":362.4385792177323,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":"180.64572957278537","output_tput_per_gpu":"181.79284964494693","p50_itl":"2.3277738749748096","p90_itl":2.594377530971542,"mean_itl":"2.2016222370772534","p50_e2el":9.456829015456606,"p50_tpot":0.01005428207731748,"p50_ttft":0.1602477750275284,"p90_e2el":"12.448685503017625","p90_tpot":"0.013336146039177195","p90_ttft":0.5262353247147985,"mean_e2el":9.940529101951688,"mean_tpot":"0.010422230131049731","mean_ttft":0.2809118332748767,"p50_intvty":99.46010986264311,"p90_intvty":74.98418186651001,"mean_intvty":95.94875448209658},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":128,"date":"2026-03-23","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1331580.9654307596","tput_per_gpu":"1824.2659226401404","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":2433.336776904707,"output_tput_per_gpu":606.1242141110072,"p50_itl":5.353172390023246,"mean_itl":4.711808901279922,"p50_e2el":"23.062116778921336","p50_tpot":"0.023864867493629004","p50_ttft":1.073228542576544,"mean_e2el":23.29444897468893,"mean_tpot":"0.023043236521292774","mean_ttft":2.1170768351476,"p50_intvty":41.90260014085397,"mean_intvty":43.39668167168984},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":13,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"27076.346361629337","tput_per_gpu":37.09459451543219,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"224.01714661582895","output_tput_per_gpu":20.10163523357794,"p50_itl":"1.3468476959969848","mean_itl":"1.2083276187868601","p50_e2el":6.154064772534184,"p50_tpot":0.005720018712256168,"p50_ttft":0.7743423988576978,"mean_e2el":6.315337552683643,"mean_tpot":0.005547099903389607,"mean_ttft":"1.2496566695322355","p50_intvty":"174.82460290860942","mean_intvty":"180.27438074243818"},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":44,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":67995.01055303759,"tput_per_gpu":93.15316445766149,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":557.2208806929847,"output_tput_per_gpu":50.96519025445029,"p50_itl":"2.0108163801487535","mean_itl":"1.7882222674758934","p50_e2el":8.573992217425257,"p50_tpot":0.008567553439737122,"p50_ttft":"0.28457346255891025","mean_e2el":8.709044318647251,"mean_tpot":0.008366554389502286,"mean_ttft":0.9668549947900995,"p50_intvty":"116.71943537135182","mean_intvty":119.5235163061539},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":88,"date":"2026-03-23","num_gpus":96,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"110953.09988529616","tput_per_gpu":"152.00574684285573","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":88,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":913.2566097678172,"output_tput_per_gpu":82.80112294058651,"p50_itl":"2.5276487809605896","mean_itl":"2.2571328926396523","p50_e2el":"10.602343758917414","p50_tpot":"0.010783262121326517","p50_ttft":0.2943238376174122,"mean_e2el":"10.626159991114136","mean_tpot":"0.010534843400016886","mean_ttft":0.938443195271909,"p50_intvty":92.73631566669026,"mean_intvty":94.92310061280998},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-23","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1088177.4229931901","tput_per_gpu":"1490.8030695006705","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"2650.8627670103006","output_tput_per_gpu":330.7433719910404,"p50_itl":4.232507979613729,"mean_itl":"3.7117418541156133","p50_e2el":"20.792238746071234","p50_tpot":"0.019016563935232572","p50_ttft":"3.4724854715168476","mean_e2el":"20.998850899078754","mean_tpot":0.01833847683992327,"mean_ttft":4.122024992261868,"p50_intvty":"52.585735436004256","mean_intvty":"54.530155842767584"},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-03-23","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":1511295.078512614,"tput_per_gpu":2070.474257562281,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":"2453.8331570085334","output_tput_per_gpu":920.3975592235239,"p50_itl":7.128685398958623,"mean_itl":6.261593289478572,"p50_e2el":"30.390580075560138","p50_tpot":0.0318410184267781,"p50_ttft":"1.0001772900577635","mean_e2el":30.67164540439353,"mean_tpot":"0.030619346679500455","mean_ttft":"2.4773543244737084","p50_intvty":"31.406030629943864","mean_intvty":"32.659090034389806"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":1024,"date":"2026-06-12","num_gpus":64,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"3077189.6212698156","tput_per_gpu":5846.66028041265,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":"2924.4433748929664","output_tput_per_gpu":2922.216905519683,"p50_itl":7.655282792984508,"p90_itl":7.711644272017293,"mean_itl":6.711062567132991,"p50_e2el":38.66795826848829,"p50_tpot":0.03240776437167865,"p50_ttft":8.710653852496762,"p90_e2el":46.29052664281335,"p90_tpot":"0.038693496149297665","p90_ttft":"10.916152420360595","mean_e2el":38.94549245645662,"mean_tpot":0.03216423692761027,"mean_ttft":9.346036482264799,"p50_intvty":"30.856802972620542","p90_intvty":25.84413659963759,"mean_intvty":31.09043134617581},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-12","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":116386.582310603,"tput_per_gpu":221.1345063901457,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":"111.16489783011403","output_tput_per_gpu":"109.96960856003167","p50_itl":1.811202552053146,"p90_itl":2.01164388039615,"mean_itl":1.754806833117535,"p50_e2el":7.159832566452678,"p50_tpot":0.008053442427364787,"p50_ttft":"0.14866087649716064","p90_e2el":11.36766081613023,"p90_tpot":"0.010400518822691312","p90_ttft":0.2685637326678261,"mean_e2el":8.010735235607717,"mean_tpot":0.008402597448914719,"mean_ttft":"0.33345573021215386","p50_intvty":124.1705033616557,"p90_intvty":96.1490495857045,"mean_intvty":119.0108185093599},{"ep":4,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-12","num_gpus":16,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":763179.8276806235,"tput_per_gpu":"1450.0416725931846","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":725.5459800499299,"output_tput_per_gpu":724.4956925432547,"p50_itl":4.54444974497892,"p90_itl":4.818315184034873,"mean_itl":4.082236433377749,"p50_e2el":"19.175553192500956","p50_tpot":"0.018553637691347106","p50_ttft":1.751761051476933,"p90_e2el":"23.907709729473574","p90_tpot":0.02312801174115255,"p90_ttft":"3.8986581749865072","mean_e2el":19.55541796295929,"mean_tpot":"0.018976486976255393","mean_ttft":"2.1038573255320445","p50_intvty":53.89778633364021,"p90_intvty":43.23761208667419,"mean_intvty":"52.696792680924794"},{"ep":4,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-12","num_gpus":16,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"1309788.6061921522","tput_per_gpu":"2488.5983517650893","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":"1245.9821013593423","output_tput_per_gpu":1242.616250405747,"p50_itl":5.34742077998817,"p90_itl":5.740484546951484,"mean_itl":4.848701485136551,"p50_e2el":"22.365120990027208","p50_tpot":"0.021960609297635394","p50_ttft":"1.9167421425227076","p90_e2el":27.69498008199735,"p90_tpot":"0.027678626331903725","p90_ttft":3.428818503452931,"mean_e2el":"22.750803870292838","mean_tpot":0.02251793120127706,"mean_ttft":2.068061705856053,"p50_intvty":45.53607718469245,"p90_intvty":36.12896059250424,"mean_intvty":"44.409052992545206"},{"ep":8,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":512,"date":"2026-06-12","num_gpus":64,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":2416026.252602892,"tput_per_gpu":4590.449879945495,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781247600,"serving_config":"agg","input_tput_per_gpu":2294.935467544399,"output_tput_per_gpu":2295.514412401096,"p50_itl":5.650088653492276,"p90_itl":6.14890466312645,"mean_itl":5.009075238074811,"p50_e2el":24.45659387658816,"p50_tpot":"0.023896236045845967","p50_ttft":2.036805893003475,"p90_e2el":"30.107044642034456","p90_tpot":"0.028784946814294946","p90_ttft":3.97471441384405,"mean_e2el":"24.671357541256747","mean_tpot":"0.023929303067818965","mean_ttft":2.637296559996571,"p50_intvty":41.84759466224959,"p90_intvty":"34.740380326267896","mean_intvty":"41.789767013517334"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":333,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":689070.626603615,"tput_per_gpu":"1460.8297283996637","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":6570.840233312593,"output_tput_per_gpu":822.0784152855475,"p50_itl":2.355745990993455,"mean_itl":2.046460177921024,"p50_e2el":"10.493492917448748","p50_tpot":0.008320912697298087,"p50_ttft":"2.5023648465285078","mean_e2el":"10.942782097102063","mean_tpot":0.008729900298306665,"mean_ttft":"2.9038195939834215","p50_intvty":120.1791241391961,"mean_intvty":"114.54884544259566"},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"154526.17151965984","tput_per_gpu":"327.59548362167885","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"2035.4088784717705","output_tput_per_gpu":"42.959917813330264","p50_itl":"0.017200859496369958","mean_itl":0.01731890968618262,"p50_e2el":6.188849315978587,"p50_tpot":"0.0049342165065338695","p50_ttft":"1.2741082410793751","mean_e2el":6.810194568615406,"mean_tpot":"0.0055208793165384285","mean_ttft":"1.6989514714921825","p50_intvty":202.6664210368159,"mean_intvty":"181.13056682916164"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-03","num_gpus":12,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":2476243.545591611,"tput_per_gpu":5249.636316654215,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"13998.674999330793","output_tput_per_gpu":875.1169753159262,"p50_itl":"1.0574833799619228","mean_itl":"1.9452113589929831","p50_e2el":"31.786046884488314","p50_tpot":0.00956597745693991,"p50_ttft":"21.961425183457322","mean_e2el":"31.566958681454434","mean_tpot":"0.010857573510096126","mean_ttft":"21.570396048758994","p50_intvty":"104.53714787656348","mean_intvty":92.1016099103663},{"ep":8,"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-03-23","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"1697750.0129847892","tput_per_gpu":2325.917517789161,"model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":"2756.6677014945276","output_tput_per_gpu":"1033.6669666730618","p50_itl":7.20045178651344,"mean_itl":6.500622894654491,"p50_e2el":"54.210403387551196","p50_tpot":0.04002017787985658,"p50_ttft":"17.608300240593962","mean_e2el":"54.897621553018965","mean_tpot":0.03918538420741825,"mean_ttft":"18.809007508284548","p50_intvty":"24.987395183551435","mean_intvty":25.51971915617171},{"ep":8,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":144,"date":"2026-03-23","num_gpus":80,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP8","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"179663.42858651077","tput_per_gpu":"246.13889716351974","model_display":"DeepSeek-R1","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1774249200,"num_decode_gpu":72,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"1232.1161136431724","output_tput_per_gpu":"136.58587311022498","p50_itl":"3.0903065650491044","mean_itl":2.778947502330743,"p50_e2el":"12.873184332042001","p50_tpot":"0.013292727736551171","p50_ttft":"0.31390740850474685","mean_e2el":"12.925549696154324","mean_tpot":0.01295201394481289,"mean_ttft":"1.0339703663260378","p50_intvty":75.22910420035822,"mean_intvty":77.20807005465639},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-15","num_gpus":28,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4505281.678175167,"tput_per_gpu":9551.197157731354,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":"11149.846268833646","output_tput_per_gpu":8352.210324404636,"p50_itl":5.170004823477939,"mean_itl":4.597570581597332,"p50_e2el":27.30202508153161,"p50_tpot":"0.027004412366538957","p50_ttft":"1.6967013999819756","mean_e2el":28.49005466069027,"mean_tpot":0.02683335181663959,"mean_ttft":"3.7952963321803184","p50_intvty":37.0309853970048,"mean_intvty":37.26705507509097},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":90,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"216139.24203300723","tput_per_gpu":458.2151931099753,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"2063.8089833224312","output_tput_per_gpu":257.5159693334183,"p50_itl":"2.0467672490049154","mean_itl":"1.7847762235442652","p50_e2el":9.342519718920812,"p50_tpot":0.007254457149061958,"p50_ttft":"2.7076359684579074","mean_e2el":9.496462341739226,"mean_tpot":0.007286461966933001,"mean_ttft":2.789670039847131,"p50_intvty":"137.84628945382985","mean_intvty":"137.24081790835416"},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":25,"date":"2026-06-15","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"104334.32389675762","tput_per_gpu":"221.18876666112615","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":665.1951314045417,"output_tput_per_gpu":"132.38749371244305","p50_itl":"1.8264863929944113","mean_itl":"1.5932300670363118","p50_e2el":7.937835168559104,"p50_tpot":"0.0065020535373848595","p50_ttft":"1.5983677944750525","mean_e2el":8.110499807374087,"mean_tpot":0.00680025032512515,"mean_ttft":"1.8777839334802702","p50_intvty":"153.79756476170178","mean_intvty":"147.05341012304518"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":4096,"date":"2026-06-03","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5487707.707961374,"tput_per_gpu":"11633.940340878113","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":"13788.836786600086","output_tput_per_gpu":5169.251003712194,"p50_itl":1.709566897014156,"mean_itl":"3.5249994032292067","p50_e2el":84.82150449592154,"p50_tpot":0.02266652452349749,"p50_ttft":62.48015016503632,"mean_e2el":84.40432881573602,"mean_tpot":"0.024277985922752123","mean_ttft":62.06131671068969,"p50_intvty":"44.117923723301274","mean_intvty":"41.189578212204566"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-06-03","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":3440833.134963053,"tput_per_gpu":7294.566246121672,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"12968.234049552353","output_tput_per_gpu":"1620.8984426909906","p50_itl":"1.3160747655201703","mean_itl":2.405085528375365,"p50_e2el":"32.083412568084896","p50_tpot":"0.011481770045444633","p50_ttft":"19.633636169834062","mean_e2el":33.0487133824372,"mean_tpot":"0.013045159165179282","mean_ttft":"21.035499178599274","p50_intvty":87.09458524617882,"mean_intvty":76.65678795773105},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-03","num_gpus":20,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1441325.4519222395","tput_per_gpu":"3055.6099580751475","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13580.14414322785,"output_tput_per_gpu":"424.47641178697165","p50_itl":0.7970074745826423,"mean_itl":"1.4635452816840842","p50_e2el":"32.361331379041076","p50_tpot":0.007337072726630762,"p50_ttft":"24.580133193638176","mean_e2el":"31.706859047013314","mean_tpot":0.008444514300374942,"mean_ttft":"23.934061629186726","p50_intvty":136.294137629355,"mean_intvty":"118.42007301184856"},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"450117.60084407683","tput_per_gpu":954.2493137894428,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":5932.854084780947,"output_tput_per_gpu":"124.48185195752535","p50_itl":"0.020272064954042435","mean_itl":0.02014959119564498,"p50_e2el":8.882847064873204,"p50_tpot":0.005908376211860631,"p50_ttft":2.691169574391097,"mean_e2el":9.341405581960862,"mean_tpot":0.00655682744139866,"mean_ttft":"3.2744267459434924","p50_intvty":"169.25123995871715","mean_intvty":152.5127828873725},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":1024,"date":"2026-06-03","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4597214.467291021,"tput_per_gpu":9746.094670656963,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":"12994.655512625572","output_tput_per_gpu":3248.972986719746,"p50_itl":"1.6842793959658593","mean_itl":"3.0710790361308566","p50_e2el":32.01607885048725,"p50_tpot":"0.014648364950596734","p50_ttft":"16.997614051564597","mean_e2el":32.66325046133882,"mean_tpot":0.01664297484682764,"mean_ttft":17.34474432911056,"p50_intvty":68.267004773066,"mean_intvty":60.08541196531415},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-03","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":754412.1020806549,"tput_per_gpu":"1599.3536564109884","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":9953.584755957054,"output_tput_per_gpu":206.981806486644,"p50_itl":0.02377890655770898,"mean_itl":"0.024056054856895766","p50_e2el":10.56475968612358,"p50_tpot":0.006825980686208835,"p50_ttft":3.551392938941717,"mean_e2el":"11.121606130302098","mean_tpot":0.007537346633171316,"mean_ttft":4.185418688417849,"p50_intvty":"146.49909602299246","mean_intvty":"132.67268293049872"},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-06-03","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":4,"tput_per_mw":91214.42909214005,"tput_per_gpu":"193.37458967533692","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":343.5495105528048,"output_tput_per_gpu":"43.199668797869016","p50_itl":0.01566619100049138,"mean_itl":"0.015693818104230393","p50_e2el":5.056824533035979,"p50_tpot":0.004086980447890385,"p50_ttft":1.024305209517479,"mean_e2el":5.382431071391329,"mean_tpot":0.004300664596998098,"mean_ttft":1.371091685583815,"p50_intvty":"244.67941864419228","mean_intvty":"232.52220149834724"},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":"121122.02125379322","tput_per_gpu":165.9371691176967,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":82.66102999936739,"output_tput_per_gpu":83.27613911832931,"p50_itl":0.0109876892529428,"p90_itl":0.01136581040918827,"mean_itl":"0.011611188216187037","p50_e2el":"11.014772782567888","p50_tpot":0.0116613756346238,"p50_ttft":"0.13332476885989308","p90_e2el":11.92966790953651,"p90_tpot":"0.011836104310678252","p90_ttft":0.2619724915362906,"mean_e2el":"10.943786948826164","mean_tpot":"0.011606839493559442","mean_ttft":"0.16773361262166872","p50_intvty":85.75317624027984,"p90_intvty":84.48725811733712,"mean_intvty":86.15609792440856},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":167,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"374888.60136057204","tput_per_gpu":794.7638348844127,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"3582.6101511315505","output_tput_per_gpu":"446.28304535352055","p50_itl":"2.2360554800834507","mean_itl":1.941370274269272,"p50_e2el":9.700842658989131,"p50_tpot":0.007865151654021511,"p50_ttft":"2.2970059184590355","mean_e2el":"10.020302153391413","mean_tpot":0.008250721397595025,"mean_ttft":"2.4443082417097934","p50_intvty":"127.14313009955661","mean_intvty":"121.20152309245184"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":615,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":1166890.7648407,"tput_per_gpu":2473.808421462284,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"11133.302027657166","output_tput_per_gpu":"1391.3717206879237","p50_itl":"2.4630343429744244","mean_itl":2.141753953961352,"p50_e2el":"11.258119062986225","p50_tpot":0.00869032832699789,"p50_ttft":3.013851944473572,"mean_e2el":"11.850302527694115","mean_tpot":0.009115749842786888,"mean_ttft":"3.4574131574339377","p50_intvty":"115.07045100854714","mean_intvty":"109.70024597496827"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-15","num_gpus":48,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"3757033.8660422144","tput_per_gpu":"7964.9117960094945","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":11954.64020199441,"output_tput_per_gpu":5970.047593017036,"p50_itl":"3.5304959309869446","mean_itl":3.192473990999287,"p50_e2el":"18.942791162000503","p50_tpot":"0.018558703577924843","p50_ttft":"1.5304136075428687","mean_e2el":"19.822094613320747","mean_tpot":0.01862623222018728,"mean_ttft":"2.6798232272704636","p50_intvty":53.88307409519043,"mean_intvty":53.68772321630302},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":1229,"date":"2026-06-15","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1991748.7501818098","tput_per_gpu":4222.507350385436,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"10558.911335426648","output_tput_per_gpu":"2638.4063541251335","p50_itl":"2.9104698229348287","mean_itl":"2.5107427418632997","p50_e2el":"11.935332480003126","p50_tpot":0.01020779047177119,"p50_ttft":"2.1814305121079087","mean_e2el":"12.577069340461689","mean_tpot":"0.010720352970702892","mean_ttft":2.707746299950556,"p50_intvty":97.96439325096046,"mean_intvty":93.28051070079961},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8192,"date":"2026-06-15","num_gpus":16,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5587543.710053062,"tput_per_gpu":"11845.592665312492","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"11849.993902118465","output_tput_per_gpu":"11841.191428506518","p50_itl":"14.513413675012998","mean_itl":12.55027234312114,"p50_e2el":76.04893743101275,"p50_tpot":0.07699958096903584,"p50_ttft":"2.2581479385262355","mean_e2el":77.58944600058156,"mean_tpot":0.07324075222660623,"mean_ttft":"10.177539545207035","p50_intvty":"12.987083662210242","mean_intvty":"13.653600892929239"},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":10,"date":"2026-06-15","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"37084.415526573706","tput_per_gpu":78.61896091633625,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"236.97685710523623","output_tput_per_gpu":"46.947381678556255","p50_itl":1.334057432017289,"mean_itl":"1.1577961430253694","p50_e2el":9.533745764929336,"p50_tpot":0.004763019833686358,"p50_ttft":4.909794491541106,"mean_e2el":9.2149844099686,"mean_tpot":0.004942025652085927,"mean_ttft":4.688001686892239,"p50_intvty":"209.95083684672505","mean_intvty":"202.34617753914748"},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":15,"date":"2026-06-15","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":64371.27680093575,"tput_per_gpu":136.4671068179838,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":412.7232333794979,"output_tput_per_gpu":81.21588150568097,"p50_itl":"1.5179410870186985","mean_itl":"1.3190271338342274","p50_e2el":8.013669793959707,"p50_tpot":0.005453614649820259,"p50_ttft":2.676443565986119,"mean_e2el":8.011980365139122,"mean_tpot":0.005736578388522181,"mean_ttft":2.779839591012957,"p50_intvty":"183.36462405405894","mean_intvty":"174.31993991415032"},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":8192,"date":"2026-06-15","num_gpus":28,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4982356.050025825,"tput_per_gpu":10562.59482605475,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":"12327.605924983412","output_tput_per_gpu":9238.836501858255,"p50_itl":6.612609301926568,"mean_itl":5.988465756964169,"p50_e2el":45.12646054546349,"p50_tpot":"0.035824721974575695","p50_ttft":"12.791490473551676","mean_e2el":48.70261326731249,"mean_tpot":0.03495267088787648,"mean_ttft":"16.530253657174775","p50_intvty":"27.913684876875976","mean_intvty":"28.610116898015235"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":8192,"date":"2026-06-03","num_gpus":40,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5770501.649172632,"tput_per_gpu":"12233.463496245979","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":32,"input_tput_per_gpu":"13593.025818967644","output_tput_per_gpu":6795.214205359314,"p50_itl":2.20726807275787,"mean_itl":4.351190301422624,"p50_e2el":"132.30861330777407","p50_tpot":"0.037082161998018774","p50_ttft":97.34448988270015,"mean_e2el":"131.33281942698935","mean_tpot":"0.037426028956807855","mean_ttft":96.88106178618874,"p50_intvty":26.9671439344186,"mean_intvty":26.71937226239169},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":74410.64327174518,"tput_per_gpu":101.9425812822909,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":51.2193177138127,"output_tput_per_gpu":50.7232635684782,"p50_itl":0.009146798402071,"p90_itl":0.009388666227459907,"mean_itl":0.009428594780951188,"p50_e2el":8.865150200203061,"p50_tpot":0.009456463676693058,"p50_ttft":"0.13219041237607598","p90_e2el":9.687439985852688,"p90_tpot":0.009554688798044329,"p90_ttft":"0.16278559956699612","mean_e2el":8.78858279744163,"mean_tpot":0.009426644845029749,"mean_ttft":"0.14515441292896866","p50_intvty":"105.74777572134678","p90_intvty":"104.66065626383161","mean_intvty":"106.08228234325128"},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":"183266.05321704815","tput_per_gpu":"251.07449290735593","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":"126.20190202160606","output_tput_per_gpu":"124.87259088574987","p50_itl":0.01443506684154272,"p90_itl":"0.014863688545301557","mean_itl":0.01547464732316056,"p50_e2el":"14.300050313468091","p50_tpot":"0.015595146214031567","p50_ttft":"0.11615805851761252","p90_e2el":"15.855537527659909","p90_tpot":"0.015854591146662673","p90_ttft":"0.18949703541584312","mean_e2el":"14.333988135671826","mean_tpot":0.01546788114585573,"mean_ttft":"0.17633013413433218","p50_intvty":64.12251519003141,"p90_intvty":"63.073212721130055","mean_intvty":64.65009593559797},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":32,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":"258321.67715262432","tput_per_gpu":353.9006976990953,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":"176.67003538341626","output_tput_per_gpu":"177.23066231567904","p50_itl":"0.019492346793413162","p90_itl":0.02033489104360342,"mean_itl":"0.021793789254962656","p50_e2el":"20.371156932786107","p50_tpot":"0.021907003378284932","p50_ttft":"0.13636806095018983","p90_e2el":"22.446337271854283","p90_tpot":"0.022765122482360414","p90_ttft":"0.35656634457409897","mean_e2el":"20.369712521578187","mean_tpot":"0.021787442159037246","mean_ttft":0.2607876803464023,"p50_intvty":45.64750288901852,"p90_intvty":43.92684470618822,"mean_intvty":"45.897999072149396"},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-30","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"INT4","tput_per_mw":380489.0664025206,"tput_per_gpu":521.2700209714532,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780124400,"serving_config":"agg","input_tput_per_gpu":260.693754442443,"output_tput_per_gpu":260.5762665290102,"p50_itl":0.02568083070218563,"p90_itl":0.02702735736966133,"mean_itl":"0.029652798800074973","p50_e2el":27.86680944217369,"p50_tpot":0.03009044068373997,"p50_ttft":0.147458853200078,"p90_e2el":"30.800893590040506","p90_tpot":0.03103253799558304,"p90_ttft":0.3490590328350693,"mean_e2el":"27.663349962569193","mean_tpot":0.02963276827548425,"mean_ttft":0.3586359859822551,"p50_intvty":33.23314571927728,"p90_intvty":32.22424154100233,"mean_intvty":"33.746425264875406"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":2253,"date":"2026-06-15","num_gpus":44,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2727633.9037977215","tput_per_gpu":5782.58387605117,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":"10600.940086413846","output_tput_per_gpu":3975.700297165167,"p50_itl":3.456942449498456,"mean_itl":3.012462033357853,"p50_e2el":"13.983722077973653","p50_tpot":0.01231116675807474,"p50_ttft":"1.4789765500463545","mean_e2el":15.42434356023337,"mean_tpot":"0.012845079540503178","mean_ttft":"3.5924411067637076","p50_intvty":81.22706967185808,"mean_intvty":77.85082193121454},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-15","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"21174.066453358595","tput_per_gpu":44.88902088112022,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781506800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"201.36126098777905","output_tput_per_gpu":"25.329990867787867","p50_itl":"1.2762638080166653","mean_itl":"1.1033146499322368","p50_e2el":"7.9454655584995635","p50_tpot":0.004481536805272189,"p50_ttft":"3.9221377320354804","mean_e2el":8.70678044076485,"mean_tpot":0.004699113913116523,"mean_ttft":4.348687794731814,"p50_intvty":"223.13774123724158","mean_intvty":"212.80607759022914"},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":512,"date":"2026-07-30","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":3818451.409687961,"tput_per_gpu":7255.057678407125,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"12897.995966053251","output_tput_per_gpu":"1612.1193907609997","p50_itl":"1.5717418513959274","mean_itl":"2.2194849229306066","p50_e2el":33.9472275020089,"p50_tpot":0.01556169670948461,"p50_ttft":"18.875405922997743","mean_e2el":33.2404075666068,"mean_tpot":0.0161409227834976,"mean_ttft":"18.381996488198862","p50_intvty":64.26034504261452,"mean_intvty":"61.954326491320266"},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-07-30","num_gpus":12,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2667436.0012146267","tput_per_gpu":5068.128402307791,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"13514.666174818287","output_tput_per_gpu":844.8595160525423,"p50_itl":"1.2634256619494408","mean_itl":"1.7943623617921154","p50_e2el":"31.221777604077943","p50_tpot":"0.012631354326894254","p50_ttft":"18.848255358519964","mean_e2el":"31.955483797465785","mean_tpot":"0.013135593314777993","mean_ttft":"19.858257328400306","p50_intvty":79.1680744693254,"mean_intvty":76.12903171073101},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":256,"date":"2026-07-30","num_gpus":20,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1620396.8729692034","tput_per_gpu":"3078.7540586414866","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":13683.00420261653,"output_tput_per_gpu":427.691522647726,"p50_itl":"1.1332159534977109","mean_itl":"1.6030485484348913","p50_e2el":"31.725231921998784","p50_tpot":"0.011094320592090812","p50_ttft":"20.894332185998792","mean_e2el":31.69981558833947,"mean_tpot":0.0115753844916939,"mean_ttft":"21.044552342593015","p50_intvty":90.13620903590115,"mean_intvty":86.39021889230253},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":2048,"date":"2026-07-30","num_gpus":32,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4710759.624564241,"tput_per_gpu":8950.443286672058,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":24,"input_tput_per_gpu":"10606.854953639433","output_tput_per_gpu":3981.208285769932,"p50_itl":"2.1522925019962713","mean_itl":2.834285097980841,"p50_e2el":54.25467871146975,"p50_tpot":"0.021437867170340477","p50_ttft":33.62573178205639,"mean_e2el":54.30629381740605,"mean_tpot":0.02203987033757777,"mean_ttft":34.00115993087293,"p50_intvty":46.64643138490525,"mean_intvty":"45.372317744311296"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":666,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1047018.1896746398","tput_per_gpu":"2219.6785621102363","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":10002.71265578684,"output_tput_per_gpu":"1246.7993004006607","p50_itl":"1.1976373725337908","mean_itl":"1.1454662726774374","p50_e2el":13.71704436186701,"p50_tpot":"0.012070727037111015","p50_ttft":"2.2639850485138595","mean_e2el":"14.584715410713429","mean_tpot":"0.012089828332834314","mean_ttft":"3.4533976601315772","p50_intvty":82.84505124882172,"mean_intvty":82.71416040574681},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":24,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"65066.299486498174","tput_per_gpu":"137.94055491137613","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":622.4512075455463,"output_tput_per_gpu":77.37672333210487,"p50_itl":0.628494206815958,"mean_itl":0.5977060778618299,"p50_e2el":8.278713780455291,"p50_tpot":0.006296191154708534,"p50_ttft":2.416962967487052,"mean_e2el":8.58886448754444,"mean_tpot":0.006311967196984088,"mean_ttft":2.798586990784194,"p50_intvty":"158.82618164351024","mean_intvty":"158.42921371293068"},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":30,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"105093.32180787134","tput_per_gpu":"222.79784223268723","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":668.731468581642,"output_tput_per_gpu":"133.61111696289626","p50_itl":0.7204494790639728,"mean_itl":0.6834093139299297,"p50_e2el":9.73743785195984,"p50_tpot":"0.0072113116552478835","p50_ttft":3.137272220104933,"mean_e2el":9.868754026909203,"mean_tpot":"0.0072257440508618756","mean_ttft":"3.2100694242243963","p50_intvty":"138.67102793598866","mean_intvty":138.3940522887358},{"ep":4,"hw":"GB300 NVL72","tp":4,"isl":1024,"osl":1024,"conc":60,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":175749.0638132687,"tput_per_gpu":"372.58801528412965","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"1118.8431964769227","output_tput_per_gpu":"223.33697904557107","p50_itl":0.8769973290618509,"mean_itl":0.8295367688757994,"p50_e2el":"11.702219558996148","p50_tpot":0.008786612287963612,"p50_ttft":"3.6655160235241055","mean_e2el":"11.972332686618513","mean_tpot":0.008767716008656876,"mean_ttft":3.891262145124298,"p50_intvty":"113.80950555538399","mean_intvty":"114.05478907079583"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-30","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"184235.59157228874","tput_per_gpu":"350.04762398734863","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"2174.9080111696867","output_tput_per_gpu":45.9042261236256,"p50_itl":"0.014410581963602453","mean_itl":"0.014172737975134152","p50_e2el":5.970159956545103,"p50_tpot":0.005286552180194451,"p50_ttft":0.7497390829958022,"mean_e2el":6.492735218662711,"mean_tpot":0.00543734377839175,"mean_ttft":"1.4624762813356937","p50_intvty":"189.15920356303334","mean_intvty":"183.91332988251455"},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":1024,"date":"2026-07-30","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4202595.190386224,"tput_per_gpu":7984.930861733825,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":"10646.461926207532","output_tput_per_gpu":"2661.8687327864086","p50_itl":"1.8693808289972367","mean_itl":"2.6216734800207013","p50_e2el":41.51070675200026,"p50_tpot":"0.018225774175410145","p50_ttft":"23.981834394500765","mean_e2el":40.97221758214375,"mean_tpot":"0.018941349558069316","mean_ttft":"23.544233583433332","p50_intvty":"54.867353802132605","mean_intvty":52.7945486109243},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-30","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":841501.0886263077,"tput_per_gpu":"1598.8520683899847","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":9950.46312062667,"output_tput_per_gpu":"206.91689301720385","p50_itl":"0.021321412990801036","mean_itl":0.02171537097356658,"p50_e2el":"10.840822547965217","p50_tpot":0.00805658699938727,"p50_ttft":"3.0273493489366956","mean_e2el":"11.329434976891388","mean_tpot":0.008402739584587085,"mean_ttft":3.594352051183705,"p50_intvty":"124.12203828693877","mean_intvty":"119.00880539416842"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-30","num_gpus":28,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":561215.5857516376,"tput_per_gpu":"1066.3096129281114","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":24,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"6629.5665622009965","output_tput_per_gpu":"139.10012138263042","p50_itl":0.01737806503660977,"mean_itl":"0.017520339783052476","p50_e2el":8.06807503750315,"p50_tpot":0.006482640526967988,"p50_ttft":"1.9214555880171247","mean_e2el":8.380395102724288,"mean_tpot":"0.0067493535905586545","mean_ttft":"2.1417854457544308","p50_intvty":154.2581292052164,"mean_intvty":"148.16233681975882"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-30","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":1,"prefill_tp":4,"tput_per_mw":"107635.60939181366","tput_per_gpu":"204.50765784444596","model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":363.3285318134055,"output_tput_per_gpu":45.68678387548642,"p50_itl":"0.012931571051012725","mean_itl":"0.012939900913895387","p50_e2el":5.172610433481168,"p50_tpot":0.005206003663678547,"p50_ttft":"0.32396186247933656","mean_e2el":5.089787815697491,"mean_tpot":0.005114809446300667,"mean_ttft":0.3537844012025744,"p50_intvty":192.0859193735955,"mean_intvty":"195.51070484615204"},{"ep":8,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":3072,"date":"2026-07-30","num_gpus":40,"sequence":"8K/1K","framework":"Dynamo SGLang","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5108581.962134122,"tput_per_gpu":9706.305728054831,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":32,"input_tput_per_gpu":"10784.910527105414","output_tput_per_gpu":5391.886531852499,"p50_itl":"1.8122211690060794","mean_itl":"2.9704723552863967","p50_e2el":60.93608592700912,"p50_tpot":0.02879577982112455,"p50_ttft":34.49340525001753,"mean_e2el":"60.979140450145344","mean_tpot":0.0286640683375388,"mean_ttft":34.59299157121943,"p50_intvty":34.72731095361415,"mean_intvty":"34.886883055968305"},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":48167.87524495283,"tput_per_gpu":65.98998908558538,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"32.878201282144566","output_tput_per_gpu":33.11178780344081,"p50_itl":0.007385444361716509,"p90_itl":0.007726094871759415,"mean_itl":0.007444628266138434,"p50_e2el":6.844503028318286,"p50_tpot":0.00743775693853043,"p50_ttft":"0.10028200084343553","p90_e2el":7.484535024687648,"p90_tpot":0.007504604291098541,"p90_ttft":"0.11412546373903751","mean_e2el":6.95620541870594,"mean_tpot":0.007444882543306547,"mean_ttft":0.1041699094697833,"p50_intvty":"134.44913678472295","p90_intvty":"133.25152948918745","mean_intvty":"134.32045357103286"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":207789.9680614113,"tput_per_gpu":"284.67225624413345","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"252.87070477865512","output_tput_per_gpu":"31.801551465478333","p50_itl":0.007609709165990353,"p90_itl":0.007708675228059292,"mean_itl":0.007615886982572456,"p50_e2el":"7.1973518542945385","p50_tpot":0.007599311401109087,"p50_ttft":"0.22458595596253872","p90_e2el":7.839914945606142,"p90_tpot":0.007670242864166618,"p90_ttft":0.2725449006073177,"mean_e2el":"7.3126173841767015","mean_tpot":0.007613991052106378,"mean_ttft":"0.23669709404930472","p50_intvty":131.590870174639,"p90_intvty":"130.37396829658942","mean_intvty":"131.33716511570555"},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":8192,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"7598495.8021037495","tput_per_gpu":"16108.811100459949","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"24198.113445406594","output_tput_per_gpu":"12064.159927986626","p50_itl":"3.8640122381038964","mean_itl":"3.6049946607212027","p50_e2el":36.9142881820444,"p50_tpot":0.03909805835512998,"p50_ttft":0.8533751624636352,"mean_e2el":37.85635652745741,"mean_tpot":0.0380621323979405,"mean_ttft":"2.8190385934018507","p50_intvty":25.57671767014466,"mean_intvty":"26.272831735883216"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":333,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":559245.105588105,"tput_per_gpu":"1185.5996238467826","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":5337.957330458489,"output_tput_per_gpu":666.5549105203194,"p50_itl":"1.1348010299989255","mean_itl":"1.0809189919059041","p50_e2el":"12.746317957993597","p50_tpot":"0.011403669309842752","p50_ttft":"1.9344247580011142","mean_e2el":"13.678862971023056","mean_tpot":"0.011416754695394257","mean_ttft":3.157918252314192,"p50_intvty":87.69107318263592,"mean_intvty":87.59056550487325},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":2253,"date":"2026-06-22","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2989977.4671514286","tput_per_gpu":"6338.7522303610285","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"15863.986625014108","output_tput_per_gpu":"3957.4436316977585","p50_itl":"1.4736939379945397","mean_itl":"1.4171433772835882","p50_e2el":15.24103235709481,"p50_tpot":0.01500289915459774,"p50_ttft":"1.0940053289523348","mean_e2el":15.63943510685309,"mean_tpot":"0.014959815853548119","mean_ttft":1.855718823860063,"p50_intvty":66.65378402503914,"mean_intvty":66.84574260737463},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-22","num_gpus":24,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":6049376.303948933,"tput_per_gpu":"12824.677764371738","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"19270.333255088288","output_tput_per_gpu":9601.850019013464,"p50_itl":"2.4604460659902543","mean_itl":2.331179526502285,"p50_e2el":"24.165554525447078","p50_tpot":"0.024854267352492708","p50_ttft":"1.1069707325659692","mean_e2el":24.73411306980783,"mean_tpot":"0.024610548825947785","mean_ttft":2.083080788802354,"p50_intvty":40.23453943814228,"mean_intvty":"40.632982509746554"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"19510.199518877893","tput_per_gpu":41.36162298002113,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":186.0431289574861,"output_tput_per_gpu":"23.276434732838013","p50_itl":0.5228369104443118,"mean_itl":0.4936988749663103,"p50_e2el":9.768830527551472,"p50_tpot":0.005213992051624471,"p50_ttft":"4.9459781049517915","mean_e2el":9.48993909531273,"mean_tpot":0.005228833906063279,"mean_ttft":4.651690278641763,"p50_intvty":"191.79162340465018","mean_intvty":"191.24722987288138"},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":1229,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1904191.9508375549","tput_per_gpu":4036.886935775616,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"18187.702219706032","output_tput_per_gpu":2268.035025284314,"p50_itl":"1.3016033419407904","mean_itl":"1.2435613367717293","p50_e2el":"14.462999968091026","p50_tpot":0.01313342991048225,"p50_ttft":"2.0422105049947277","mean_e2el":"14.858669718021988","mean_tpot":"0.013127548718300153","mean_ttft":"2.7679955028398364","p50_intvty":76.14157206579105,"mean_intvty":76.17568378214993},{"ep":32,"hw":"GB300 NVL72","tp":32,"isl":1024,"osl":1024,"conc":4301,"date":"2026-06-22","num_gpus":40,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":4629883.808593357,"tput_per_gpu":9815.353674217917,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"24580.882237862665","output_tput_per_gpu":6123.971533306729,"p50_itl":1.90565578895621,"mean_itl":"1.7998397884131152","p50_e2el":"18.708113909116946","p50_tpot":"0.019220393270071315","p50_ttft":0.9312563450075686,"mean_e2el":"19.368319064867933","mean_tpot":0.01900144712771546,"mean_ttft":1.879910803437652,"p50_intvty":52.02807174383532,"mean_intvty":52.62757058863178},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":1024,"osl":1024,"conc":12,"date":"2026-06-22","num_gpus":36,"sequence":"1K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":36030.77251545865,"tput_per_gpu":76.38523773277234,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1782111600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":345.8497179640449,"output_tput_per_gpu":42.70217770386327,"p50_itl":0.5722004710114561,"mean_itl":0.539386405127876,"p50_e2el":7.341117169504287,"p50_tpot":0.005727553641682634,"p50_ttft":"2.2145244915009243","mean_e2el":7.779836884967275,"mean_tpot":0.005727748941185004,"mean_ttft":"2.5343042540330014","p50_intvty":"174.59461099105852","mean_intvty":"174.58865782499046"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2103176.6817496335","tput_per_gpu":"2881.3520539969977","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":2559.136726510329,"output_tput_per_gpu":"322.21532748666846","p50_itl":0.01910815481096506,"p90_itl":"0.019779410026967526","mean_itl":"0.023619761253984115","p50_e2el":22.36477159243077,"p50_tpot":"0.023922653683571812","p50_ttft":"0.27133110724389553","p90_e2el":25.42959822434932,"p90_tpot":0.02491098530824671,"p90_ttft":0.6585218762979025,"mean_e2el":"22.472380074547253","mean_tpot":"0.023591871304707045","mean_ttft":0.6371657464798772,"p50_intvty":41.80138262364768,"p90_intvty":40.14293243025409,"mean_intvty":42.38748114061135},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":667445.6999467326,"tput_per_gpu":914.4006089270235,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":456.4760368771876,"output_tput_per_gpu":"457.92457204983594","p50_itl":"0.015298089012503624","p90_itl":0.01587388664484024,"mean_itl":"0.016889833386531383","p50_e2el":"15.756152363494039","p50_tpot":0.01702598087170648,"p50_ttft":"0.10365770012140274","p90_e2el":"17.371188016422092","p90_tpot":"0.017380441410942013","p90_ttft":"0.26107676094398075","mean_e2el":"15.751592535179224","mean_tpot":"0.016882769445361295","mean_ttft":0.1675017594534438,"p50_intvty":"58.733767383809585","p90_intvty":57.53593803264634,"mean_intvty":59.23198816618086},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":2151,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":9708670.694792934,"tput_per_gpu":"16601.826888095915","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":16,"input_tput_per_gpu":"22137.397180275493","output_tput_per_gpu":5530.686303736757,"p50_itl":4.466269871001714,"mean_itl":4.193287173196506,"p50_e2el":43.37132857950019,"p50_tpot":"0.044718717017721496","p50_ttft":2.143370847999904,"mean_e2el":44.00534525323752,"mean_tpot":"0.044267797517984576","mean_ttft":"3.2614489999496254","p50_intvty":"22.362001119211715","mean_intvty":"22.589784359471064"},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":30,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":562816.2470439302,"tput_per_gpu":962.4157824451207,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":5133.102423756904,"output_tput_per_gpu":128.2784541827641,"p50_itl":0.7563211830001819,"mean_itl":0.7162829572899431,"p50_e2el":10.44058839850004,"p50_tpot":0.007578686538952426,"p50_ttft":3.479817200000525,"mean_e2el":"10.527182841998323","mean_tpot":"0.0075695806186588995","mean_ttft":"3.5481994122916465","p50_intvty":131.9489854686913,"mean_intvty":132.1077151269141},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":1127,"date":"2026-07-22","num_gpus":20,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":8402438.759990789,"tput_per_gpu":"14368.170279584248","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":21284.62700492678,"output_tput_per_gpu":3993.485191570452,"p50_itl":3.085370051994687,"mean_itl":2.910194236541178,"p50_e2el":31.47325545900094,"p50_tpot":"0.030893588085740255","p50_ttft":2.927976494000177,"mean_e2el":"31.856907381081086","mean_tpot":"0.030724226037815338","mean_ttft":3.546424417469676,"p50_intvty":32.36917632308227,"mean_intvty":"32.547605878475224"},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-22","num_gpus":36,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":96991.53230571175,"tput_per_gpu":"165.85552024276708","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"1324.9968728089598","output_tput_per_gpu":"20.962851171993005","p50_itl":0.5403338299947791,"mean_itl":0.5118868017538521,"p50_e2el":5.528259518505365,"p50_tpot":0.00542826161698411,"p50_ttft":0.4069340290079708,"mean_e2el":5.468217146774987,"mean_tpot":0.005423804648537596,"mean_ttft":"0.45172669868752563","p50_intvty":184.2210399128829,"mean_intvty":"184.37242209112878"},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":5,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"173595.45872408675","tput_per_gpu":296.8482344181883,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"1582.9033500856701","output_tput_per_gpu":"39.637211284691965","p50_itl":0.5800941140005307,"mean_itl":0.5488846017574496,"p50_e2el":"5.7129023835000226","p50_tpot":0.005803966903605362,"p50_ttft":0.3361333339998964,"mean_e2el":5.711121141259946,"mean_tpot":0.005830776617105962,"mean_ttft":"0.37047417708993635","p50_intvty":172.2959514773957,"mean_intvty":"171.50374052510665"},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":195,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"2185277.9894820945","tput_per_gpu":3736.825362014382,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"19928.676584730914","output_tput_per_gpu":"498.45511747107486","p50_itl":"1.5439094949997525","mean_itl":"1.4590739188125315","p50_e2el":"17.447171898000082","p50_tpot":"0.015478323453716681","p50_ttft":3.169773960499924,"mean_e2el":17.67190759468949,"mean_tpot":"0.015404866319326832","mean_ttft":"3.4848457142215405","p50_intvty":64.60648034589808,"mean_intvty":64.91455227660154},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":615,"date":"2026-07-22","num_gpus":16,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"6545431.2612804845","tput_per_gpu":"11192.687456789628","model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"19897.601939765354","output_tput_per_gpu":2487.772973813901,"p50_itl":"2.4837234345013712","mean_itl":"2.3470633870085367","p50_e2el":"27.586152054000195","p50_tpot":"0.024874296682925068","p50_ttft":4.657701241500035,"mean_e2el":"27.874815680981634","mean_tpot":"0.024779562874710778","mean_ttft":5.052582113940733,"p50_intvty":40.20214170262144,"mean_intvty":"40.355836987768974"},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":24,"date":"2026-07-22","num_gpus":36,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":335031.6794557608,"tput_per_gpu":572.904171869351,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":4581.275575380407,"output_tput_per_gpu":71.85774643046896,"p50_itl":0.665629932000229,"mean_itl":0.6339778713782003,"p50_e2el":9.34894149449974,"p50_tpot":0.006701036297590819,"p50_ttft":"3.0959635114995763","mean_e2el":9.46391906045413,"mean_tpot":0.006693204760111082,"mean_ttft":3.272068697270815,"p50_intvty":"149.23064964735732","mean_intvty":"149.40526038581908"},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"162641.82249934025","tput_per_gpu":"222.81929682409614","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"111.95176944949776","output_tput_per_gpu":"110.86752737459838","p50_itl":0.008314144797623158,"p90_itl":0.008389848284423351,"mean_itl":0.008590959427817966,"p50_e2el":8.052786422427744,"p50_tpot":0.008547626344939423,"p50_ttft":"0.10558773763477802","p90_e2el":8.816021524742245,"p90_tpot":0.008634618206540585,"p90_ttft":"0.14303046157583593","mean_e2el":8.003611694020218,"mean_tpot":0.008588038570231058,"mean_ttft":0.1280645821709186,"p50_intvty":"116.99154357537454","p90_intvty":"115.81287974522324","mean_intvty":"116.44102338644892"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1568566.2944001572","tput_per_gpu":2148.935823328215,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":1910.819986981803,"output_tput_per_gpu":238.1158363464122,"p50_itl":"0.013352463487535715","p90_itl":"0.014169205073267221","mean_itl":0.01581859860260496,"p50_e2el":15.02005557436496,"p50_tpot":"0.015995460694273676","p50_ttft":"0.25209455052390695","p90_e2el":"16.659811503533273","p90_tpot":"0.016420186971510015","p90_ttft":0.6127600804902592,"mean_e2el":"14.956987510900944","mean_tpot":"0.015803724962490252","mean_ttft":"0.49760214287089183","p50_intvty":62.51773669501103,"p90_intvty":60.9006463650541,"mean_intvty":63.2762214208027},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":1824337.858944687,"tput_per_gpu":2499.342866754221,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"1251.9927872426374","output_tput_per_gpu":"1247.3500795115838","p50_itl":"0.022363875061273575","p90_itl":"0.023571609053760767","mean_itl":0.02383160574702554,"p50_e2el":"22.143789749476127","p50_tpot":"0.024195496786363728","p50_ttft":0.09926267201080918,"p90_e2el":24.60735154193826,"p90_tpot":"0.025050952029600303","p90_ttft":0.4236580020282419,"mean_e2el":"22.079143716174286","mean_tpot":"0.023823492985688022","mean_ttft":"0.19698296637278143","p50_intvty":"41.330004869484114","p90_intvty":39.91864256569555,"mean_intvty":"41.975372822144536"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1062423.8171978581","tput_per_gpu":"1455.5206295610656","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"1291.8955616259416","output_tput_per_gpu":163.6250679351239,"p50_itl":0.01046680100262165,"p90_itl":"0.010784883983433247","mean_itl":"0.011641347409017268","p50_e2el":"11.142788457684219","p50_tpot":0.01164506857084034,"p50_ttft":0.2414523852057755,"p90_e2el":"12.323005273286254","p90_tpot":"0.012127916736583104","p90_ttft":0.575344641879201,"mean_e2el":"11.115335852617864","mean_tpot":"0.011627665454305472","mean_ttft":"0.34985453891567886","p50_intvty":85.87325990540194,"p90_intvty":82.45439193885314,"mean_intvty":86.00178633706061},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1043891.4142767332","tput_per_gpu":"1430.1312375591244","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":715.2267858602269,"output_tput_per_gpu":714.9044516988976,"p50_itl":0.01882031001150608,"p90_itl":"0.019671556539833546","mean_itl":"0.021625316206559844","p50_e2el":"20.254731030669063","p50_tpot":0.02189340484278375,"p50_ttft":"0.10790327796712518","p90_e2el":22.39228266198188,"p90_tpot":"0.022426113471659157","p90_ttft":"0.24515701020136485","mean_e2el":20.13137295123597,"mean_tpot":"0.021610286442082516","mean_ttft":"0.21847782725963044","p50_intvty":"45.675855682612486","p90_intvty":44.5908739944504,"mean_intvty":"46.274259375510304"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":2836607.19679281,"tput_per_gpu":"3886.1518596061496","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":3455.017226604507,"output_tput_per_gpu":431.1346330016425,"p50_itl":"0.025681957602500916","p90_itl":"0.026777894236147404","mean_itl":0.03549560332377036,"p50_e2el":33.86448967503384,"p50_tpot":"0.036226608277946486","p50_ttft":0.3112456756643951,"p90_e2el":"38.223872306570414","p90_tpot":0.03765161797930359,"p90_ttft":"1.1354788580909465","mean_e2el":"33.632933421018244","mean_tpot":0.03545445912993173,"mean_ttft":0.9499685740258428,"p50_intvty":"27.604019463471708","p90_intvty":"26.559283602358917","mean_intvty":"28.205196879051233"},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":89237.77459701075,"tput_per_gpu":"122.25575119790471","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":60.7521487040179,"output_tput_per_gpu":61.50360249388682,"p50_itl":0.00789288233499974,"p90_itl":0.00793729757424444,"mean_itl":0.00807091519797998,"p50_e2el":7.66890948661603,"p50_tpot":0.007940942227932108,"p50_ttft":0.07288729201536626,"p90_e2el":8.033171174000017,"p90_tpot":0.007957295032917713,"p90_ttft":0.3552852850407362,"mean_e2el":7.507119986141333,"mean_tpot":0.007922295284491093,"mean_ttft":0.1393834167101886,"p50_intvty":"125.92964050065994","p90_intvty":"125.67084616860417","mean_intvty":"126.22604486323907"},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":115,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1453565.2197523657","tput_per_gpu":2485.596525776545,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"13256.523220608366","output_tput_per_gpu":"331.41118681018116","p50_itl":"1.1678482140000597","mean_itl":1.106129500469503,"p50_e2el":"15.562135622499909","p50_tpot":0.01170599804861638,"p50_ttft":4.768956678999984,"mean_e2el":15.64133365680826,"mean_tpot":"0.011678528269545654","mean_ttft":4.878693802668701,"p50_intvty":85.42629136335773,"mean_intvty":85.6272277567475},{"ep":4,"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":60,"date":"2026-07-22","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":914253.2710942212,"tput_per_gpu":1563.373093571118,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":20,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":8342.083030322594,"output_tput_per_gpu":"207.63110622082277","p50_itl":0.9309318710002117,"mean_itl":0.8828757009570978,"p50_e2el":12.93190623999999,"p50_tpot":0.0093285162205826,"p50_ttft":4.334186606999992,"mean_e2el":"12.930386317325008","mean_tpot":0.009316229383439319,"mean_ttft":4.373113765404169,"p50_intvty":"107.19818418641785","mean_intvty":"107.33956398471857"},{"ep":8,"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":308,"date":"2026-07-22","num_gpus":12,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":5245682.09434341,"tput_per_gpu":8970.11638132723,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1784703600,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":23922.09715373639,"output_tput_per_gpu":"1494.1259951226489","p50_itl":2.050800728000013,"mean_itl":"1.9404992052270267","p50_e2el":22.89682037349985,"p50_tpot":0.0205363822259019,"p50_ttft":3.946353267499944,"mean_e2el":23.22039018022792,"mean_tpot":"0.020476582948417563","mean_ttft":4.36604645870049,"p50_intvty":48.69406836120975,"mean_intvty":48.83627324535026},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":256,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2258308.9341111556","tput_per_gpu":"3093.8832397322826","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":1546.193701406748,"output_tput_per_gpu":"1547.6895383255346","p50_itl":0.0329332584515214,"p90_itl":0.06359007898718119,"mean_itl":0.03998972415663319,"p50_e2el":"37.409529307391495","p50_tpot":"0.040575410964398664","p50_ttft":"0.18450535694137216","p90_e2el":"41.743946958426385","p90_tpot":"0.041689759600135115","p90_ttft":0.3014730690978453,"mean_e2el":"37.279939094719886","mean_tpot":0.03994757621020178,"mean_ttft":"0.45330854108979113","p50_intvty":"24.645468184596123","p90_intvty":"23.986705838350748","mean_intvty":25.03280786644124},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":128,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"3691032.4453515294","tput_per_gpu":5056.714450131595,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":4496.604945493196,"output_tput_per_gpu":560.1095046383984,"p50_itl":0.03484778944402933,"p90_itl":"0.17364818649366498","mean_itl":0.05465383079513883,"p50_e2el":51.66614718688652,"p50_tpot":"0.055758145439432694","p50_ttft":0.4683341686613858,"p90_e2el":"59.845184174086896","p90_tpot":0.05871585947555108,"p90_ttft":1.519910278171301,"mean_e2el":51.78812752067388,"mean_tpot":0.0545716330414461,"mean_ttft":"1.5554502009385032","p50_intvty":"17.934599368737082","p90_intvty":"17.031173671508526","mean_intvty":"18.324538670860726"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"372167.11045594607","tput_per_gpu":509.8689413246461,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"452.65401683966206","output_tput_per_gpu":57.21492448498404,"p50_itl":0.008300096727907658,"p90_itl":0.008381417021155357,"mean_itl":0.008452373404688181,"p50_e2el":8.217879162169993,"p50_tpot":0.008453278903414406,"p50_ttft":"0.23906927183270454","p90_e2el":8.784930631611497,"p90_tpot":0.008519682939090507,"p90_ttft":0.3341970550827683,"mean_e2el":8.150275449361652,"mean_tpot":0.008448617346124957,"mean_ttft":0.2553363617043942,"p50_intvty":"118.29729166940004","p90_intvty":"117.37525998904742","mean_intvty":"118.36256265750514"},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":654998.4584432165,"tput_per_gpu":897.3478880672066,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":797.5250381168853,"output_tput_per_gpu":99.82284995032126,"p50_itl":0.008884469978511333,"p90_itl":0.009179641865193844,"mean_itl":0.00943699006746962,"p50_e2el":8.959557075053453,"p50_tpot":0.009441470707043952,"p50_ttft":"0.23480528173968196","p90_e2el":9.84368686536327,"p90_tpot":0.009565488775294697,"p90_ttft":0.7861975970678031,"mean_e2el":8.978728340612724,"mean_tpot":0.009429653561677248,"mean_ttft":0.3077861146535724,"p50_intvty":"105.91570222782505","p90_intvty":104.5424884698787,"mean_intvty":106.0484347022109},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-05","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"276106.68877185404","tput_per_gpu":378.26616361744,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":"188.43198823253996","output_tput_per_gpu":"189.83417538490002","p50_itl":0.009742265567183495,"p90_itl":"0.010233982931822538","mean_itl":"0.010199963439825339","p50_e2el":9.638672414701432,"p50_tpot":"0.010245991212440839","p50_ttft":"0.10509507823735476","p90_e2el":"10.454299660678952","p90_tpot":"0.010332896073158522","p90_ttft":"0.16180298291146764","mean_e2el":9.589912289055064,"mean_tpot":"0.010196474017951332","mean_ttft":"0.12358140678843484","p50_intvty":97.59914675563891,"p90_intvty":96.77828877014183,"mean_intvty":98.0731180444786},{"ep":1,"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-05","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":4433306.594196022,"tput_per_gpu":"6073.6300340485495","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780642800,"serving_config":"agg","input_tput_per_gpu":5398.588167615218,"output_tput_per_gpu":675.0418664333316,"p50_itl":0.05024711461737752,"p90_itl":0.2216530149802566,"mean_itl":0.09081530394817962,"p50_e2el":86.72998845716938,"p50_tpot":0.09297121088775495,"p50_ttft":0.6866031568497419,"p90_e2el":99.9042845448479,"p90_tpot":0.09784140585877223,"p90_ttft":"1.6750296341255306","mean_e2el":86.41183020444878,"mean_tpot":0.09063082349260669,"mean_ttft":"2.7661818072014284","p50_intvty":"10.756017808645192","p90_intvty":"10.220621742122509","mean_intvty":11.03377373683001},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-13","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"143881.66482278294","tput_per_gpu":"197.11788080721263","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":99.68642234649238,"output_tput_per_gpu":97.43145846072025,"p50_itl":0.08975566364824772,"p90_itl":0.1449015092104674,"mean_itl":0.09947401333357783,"p50_e2el":4.455150877125561,"p50_tpot":0.004752324636132815,"p50_ttft":"0.15509797260165215","p90_e2el":5.401549409143627,"p90_tpot":0.0060419254720913,"p90_ttft":"0.16133250519633294","mean_e2el":4.592697373358533,"mean_tpot":0.00486247298568468,"mean_ttft":"0.16113042263314128","p50_intvty":210.4233352235267,"p90_intvty":165.5101514606847,"mean_intvty":"205.65666954737662"},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-06-13","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":85774.53042584358,"tput_per_gpu":117.5111066834057,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":58.32496006001818,"output_tput_per_gpu":59.18614662338752,"p50_itl":0.07856282591819763,"p90_itl":"0.11822453774511817","mean_itl":0.08249013608898906,"p50_e2el":3.677994806319475,"p50_tpot":0.003898344945693329,"p50_ttft":"0.15011687576770782","p90_e2el":5.060162815637887,"p90_tpot":0.005067669915136984,"p90_ttft":0.188321751262993,"mean_e2el":"3.9437698011286555","mean_tpot":0.00400015044732562,"mean_ttft":"0.17809528713114559","p50_intvty":256.5191161712725,"p90_intvty":"197.32934795398353","mean_intvty":"249.99059739579792"},{"ep":1,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-03","num_gpus":9,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"349396.64282581664","tput_per_gpu":663.8536213690517,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":8,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":"5310.6886531340515","output_tput_per_gpu":82.99924239842663,"p50_itl":"0.17836537699986366","mean_itl":0.1756452190514178,"p50_e2el":"5.4128639195005235","p50_tpot":"0.0055736936489480145","p50_ttft":0.2282928010008618,"mean_e2el":5.374769606275095,"mean_tpot":0.005575299254994308,"mean_ttft":0.2503204449502846,"p50_intvty":"179.41423820247837","mean_intvty":"179.36256948076965"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":592714.2922567272,"tput_per_gpu":"1126.1571552877817","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":5005.010671126508,"output_tput_per_gpu":"156.44377632809997","p50_itl":"0.19160702801309526","mean_itl":"0.18800945964804672","p50_e2el":5.749576010974124,"p50_tpot":0.005973944930246175,"p50_ttft":"0.21523851645179093","mean_e2el":5.730820269818651,"mean_tpot":0.005968174425683015,"mean_ttft":"0.24564437931985594","p50_intvty":167.3935752130865,"mean_intvty":"167.55542460298608"},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"308656.84594373597","tput_per_gpu":422.85987894291827,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":"213.88236014689144","output_tput_per_gpu":"208.97751879602683","p50_itl":0.1666999962180853,"p90_itl":"0.27790135480463507","mean_itl":"0.18267831389139788","p50_e2el":8.084178095683455,"p50_tpot":0.00839560540765314,"p50_ttft":0.1534733111038804,"p90_e2el":11.26903347335756,"p90_tpot":"0.011408595021200448","p90_ttft":0.3405148889869535,"mean_e2el":8.433337793312967,"mean_tpot":0.008878738279486821,"mean_ttft":0.2881687375018373,"p50_intvty":"119.10993328586345","p90_intvty":87.65321217395417,"mean_intvty":"112.62861552191158"},{"ep":1,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-07-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"55563.449513052925","tput_per_gpu":"105.57055407480055","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":52.62770904624386,"output_tput_per_gpu":52.9428450285567,"p50_itl":"0.046414096985245124","p90_itl":0.07391357431188225,"mean_itl":0.04790512390627555,"p50_e2el":1.964887340000132,"p50_tpot":0.002110337161989842,"p50_ttft":"0.035179897502530366","p90_e2el":2.970902182953432,"p90_tpot":"0.0031074270213487906","p90_ttft":0.03598278932622634,"mean_e2el":2.181288327090442,"mean_tpot":"0.0023286963584864483","mean_ttft":0.03513888110755943,"p50_intvty":473.8579303873404,"p90_intvty":321.8096493110709,"mean_intvty":429.4248137399745},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":901710.0204862225,"tput_per_gpu":"1235.3427280661247","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":624.7349132528038,"output_tput_per_gpu":610.6078148133208,"p50_itl":"0.48080223612487316","p90_itl":0.7771133292466402,"mean_itl":0.5098206584038444,"p50_e2el":"22.267522908747196","p50_tpot":0.02344809243935559,"p50_ttft":0.2839846294373274,"p90_e2el":"30.764701644517483","p90_tpot":0.03277098004277321,"p90_ttft":0.5465504579246044,"mean_e2el":23.33098011799448,"mean_tpot":"0.024801244655471268","mean_ttft":0.5666913128632587,"p50_intvty":42.64739243016573,"p90_intvty":"30.514802996272437","mean_intvty":"40.320557048309084"},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":89417.88118705168,"tput_per_gpu":122.5024972262608,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":61.0684090650315,"output_tput_per_gpu":61.4340881612293,"p50_itl":0.08072085306048393,"p90_itl":"0.11146183647215366","mean_itl":0.08034559055981914,"p50_e2el":4.069519085809588,"p50_tpot":0.004074941704675393,"p50_ttft":"0.11986505053937435","p90_e2el":4.643231923691928,"p90_tpot":"0.0048417562343686825","p90_ttft":"0.12538996990770102","mean_e2el":"3.7596739953383804","mean_tpot":0.003941320826449076,"mean_ttft":0.1200189558789134,"p50_intvty":245.4022836333212,"p90_intvty":"206.53662671028505","mean_intvty":253.722050052177},{"ep":1,"hw":"H200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-06-13","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":50008.64495972403,"tput_per_gpu":68.51184359482191,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":34.15366531443361,"output_tput_per_gpu":34.3581782803883,"p50_itl":0.06955798156559467,"p90_itl":"0.10412715747952461","mean_itl":0.07202062763705153,"p50_e2el":3.413194777444005,"p50_tpot":0.003398532222209521,"p50_ttft":"0.12145284190773964","p90_e2el":4.05372165068984,"p90_tpot":0.00427785339488754,"p90_ttft":"0.14310993850231168","mean_e2el":"3.3611276473850014","mean_tpot":"0.0034998208495584104","mean_ttft":"0.12740166746079923","p50_intvty":294.2446722926347,"p90_intvty":233.7621016173905,"mean_intvty":"285.72891098873674"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"1076796.1468549676","tput_per_gpu":"2045.9126790244384","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":9080.703877848508,"output_tput_per_gpu":287.214879318421,"p50_itl":0.2080638816114515,"mean_itl":"0.20441428557315722","p50_e2el":6.360162809607573,"p50_tpot":0.006497416991233737,"p50_ttft":"0.23941463907249272","mean_e2el":6.31802729566698,"mean_tpot":0.006492339693282769,"mean_ttft":"0.31335774371109437","p50_intvty":153.9073144526805,"mean_intvty":"154.02767680727482"},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":128,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":6766834.063651251,"tput_per_gpu":"12856.984720937377","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":57167.60489907475,"output_tput_per_gpu":"1779.3296764030335","p50_itl":"0.44663390098139644","mean_itl":0.432682973805924,"p50_e2el":"15.634900334989652","p50_tpot":"0.013961508911630177","p50_ttft":"2.7583777974941768","mean_e2el":"15.993034874697333","mean_tpot":0.01374726290637841,"mean_ttft":3.35903015136455,"p50_intvty":71.62549594958055,"mean_intvty":72.74175279909889},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":1924893.214285759,"tput_per_gpu":3657.297107142942,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":"16261.557527066945","output_tput_per_gpu":"506.23200216194124","p50_itl":"0.23287423490546644","mean_itl":0.2284847291975266,"p50_e2el":6.99953840195667,"p50_tpot":0.007279469749593325,"p50_ttft":"0.25592951150611043","mean_e2el":7.005054452460899,"mean_tpot":0.00725264547411556,"mean_ttft":"0.37899739017011597","p50_intvty":137.3726431181153,"mean_intvty":"137.88072277473998"},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":256,"date":"2026-06-03","num_gpus":6,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":8846593.56259455,"tput_per_gpu":"16808.527768929645","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":44823.44907535463,"output_tput_per_gpu":"2801.0671157171514","p50_itl":0.6678273095749319,"mean_itl":0.6510823114128456,"p50_e2el":19.98868908197619,"p50_tpot":"0.021114279421894068","p50_ttft":0.4521596624981612,"mean_e2el":"20.386332351692182","mean_tpot":"0.020669079474670095","mean_ttft":"1.3508220087597693","p50_intvty":47.36131316719567,"mean_intvty":"48.381448299402855"},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-06-03","num_gpus":5,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"3056014.1318743955","tput_per_gpu":5806.426850561352,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":1,"input_tput_per_gpu":"25788.262298226433","output_tput_per_gpu":810.9679886450816,"p50_itl":0.28920337499585,"mean_itl":0.2830215775159221,"p50_e2el":8.712319772515912,"p50_tpot":0.009040432213780824,"p50_ttft":"0.32013985197409056","mean_e2el":8.83923948716656,"mean_tpot":0.00898916036839502,"mean_ttft":0.529902934659367,"p50_intvty":"110.61418042332595","mean_intvty":"111.24509509429811"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1236126.3190227554","tput_per_gpu":2348.640006143235,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":2087.965367448874,"output_tput_per_gpu":260.6746386943609,"p50_itl":0.06188301999645773,"p90_itl":"0.11349918829509989","mean_itl":0.07328964246936007,"p50_e2el":"3.2377525820047595","p50_tpot":0.003225035724609161,"p50_ttft":"0.16571483999723569","p90_e2el":4.527181394098443,"p90_tpot":"0.0045736610522561115","p90_ttft":"0.19631823621748495","mean_e2el":3.475651362550707,"mean_tpot":"0.0035656045227447585","mean_ttft":0.1922755676729139,"p50_intvty":310.074084565742,"p90_intvty":"218.64322444854466","mean_intvty":280.457351234851},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":434948.26646361,"tput_per_gpu":826.401706280859,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":"417.99365740401225","output_tput_per_gpu":"408.40804887684675","p50_itl":0.08878568396903574,"p90_itl":"0.13666583118028938","mean_itl":0.09408037283475856,"p50_e2el":4.011313202558085,"p50_tpot":0.004302169436935419,"p50_ttft":"0.057397139025852084","p90_e2el":"5.6028250833740465","p90_tpot":"0.0059041969909613915","p90_ttft":0.1268760807579422,"mean_e2el":4.303849263943266,"mean_tpot":0.004563334859014022,"mean_ttft":"0.11492075286223553","p50_intvty":"232.44086841738476","p90_intvty":169.3710425873118,"mean_intvty":219.1379837104624},{"ep":1,"hw":"B300","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-29","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"248694.47342945484","tput_per_gpu":472.5194995159642,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":"419.95392297281956","output_tput_per_gpu":52.56557654314466,"p50_itl":0.04192626476287842,"p90_itl":0.07130544539541006,"mean_itl":0.04636316600092791,"p50_e2el":2.098491852171719,"p50_tpot":0.002161546618584825,"p50_ttft":"0.14309501508250833","p90_e2el":"3.2481147008016706","p90_tpot":"0.0030789573797224508","p90_ttft":"0.15343933245167135","mean_e2el":2.211909174360335,"mean_tpot":"0.0022096291083156206","mean_ttft":"0.14411207297816872","p50_intvty":"462.63170611360897","p90_intvty":"324.78526873604983","mean_intvty":"452.56463912275785"},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":464012.8997799138,"tput_per_gpu":635.6976726984818,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":320.4119608760135,"output_tput_per_gpu":"315.28571182246833","p50_itl":"0.22749284282326698","p90_itl":0.3721059700474143,"mean_itl":0.2476509405089999,"p50_e2el":"10.839562912471592","p50_tpot":"0.011528950293686172","p50_ttft":0.1717239199206233,"p90_e2el":14.88623042870313,"p90_tpot":"0.015814361417849847","p90_ttft":"0.36723790708929266","mean_e2el":"11.356114305288065","mean_tpot":"0.012066193704975438","mean_ttft":"0.26444801054894923","p50_intvty":86.7381656201302,"p90_intvty":63.2336629711326,"mean_intvty":82.8761765682292},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":226287.4321537258,"tput_per_gpu":"310.01378205060433","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":156.7801190037915,"output_tput_per_gpu":"153.23366304681284","p50_itl":"0.12174089066684246","p90_itl":"0.18969606161117553","mean_itl":0.1281158847459859,"p50_e2el":5.600581619888544,"p50_tpot":0.006144199079896096,"p50_ttft":"0.14603086095303297","p90_e2el":7.090565089136362,"p90_tpot":0.007785814089027115,"p90_ttft":0.2029348559677601,"mean_e2el":5.860252970363945,"mean_tpot":0.006235164337422334,"mean_ttft":"0.16550215044990182","p50_intvty":"162.75514302132783","p90_intvty":"128.43872054552952","mean_intvty":160.3806966238532},{"ep":1,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-06-13","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":633092.6244512819,"tput_per_gpu":867.336895498256,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":436.2853335183778,"output_tput_per_gpu":431.0515619798782,"p50_itl":0.3426201120018959,"p90_itl":0.5432974848896265,"mean_itl":"0.36296577761073356","p50_e2el":"15.815853408537805","p50_tpot":"0.017172056398735525","p50_ttft":"0.19560118485242128","p90_e2el":22.41540284454823,"p90_tpot":0.02376256418414414,"p90_ttft":0.385142194479705,"mean_e2el":"16.669650723127415","mean_tpot":"0.017635937884499596","mean_ttft":0.3486678586283233,"p50_intvty":58.23414370300086,"p90_intvty":"42.083000481373226","mean_intvty":56.70239975606344},{"ep":4,"hw":"H200","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-06-13","num_gpus":16,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"1041295.9266197513","tput_per_gpu":"1426.5754194690592","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1781334000,"serving_config":"agg","input_tput_per_gpu":721.2882977528493,"output_tput_per_gpu":705.2871217162099,"p50_itl":0.8377594547346234,"p90_itl":"1.3278530221432447","mean_itl":0.8842741311702886,"p50_e2el":"38.796218923293054","p50_tpot":0.04137100320028242,"p50_ttft":0.4048118656501174,"p90_e2el":54.56769824791701,"p90_tpot":0.057110205594926,"p90_ttft":0.7549702767282737,"mean_e2el":40.53340516175085,"mean_tpot":0.04300772275101594,"mean_ttft":"1.0326035507649067","p50_intvty":24.17151924401905,"p90_intvty":"17.510005253576704","mean_intvty":"23.251637985793558"},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":512,"date":"2026-06-03","num_gpus":6,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"11043272.311915591","tput_per_gpu":"20982.217392639624","model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":55956.62745394813,"output_tput_per_gpu":"3495.0123619853703","p50_itl":"1.0426559259649366","mean_itl":0.999139767772509,"p50_e2el":32.58712705352809,"p50_tpot":"0.033385939636881315","p50_ttft":"1.2684790894854814","mean_e2el":"32.899582645739244","mean_tpot":"0.031730772243852665","mean_ttft":"3.6759153839705503","p50_intvty":"29.952728929495333","mean_intvty":"31.515148522542944"},{"ep":4,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-06-03","num_gpus":6,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"4252159.6243672995","tput_per_gpu":8079.103286297869,"model_display":"MiniMax-M2.5","spec_decoding":"off","benchmark_type":"single_turn","date_timestamp":1780470000,"num_decode_gpu":4,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"21550.556817622142","output_tput_per_gpu":"1343.3765206357325","p50_itl":"0.35608894657343626","mean_itl":0.3481319445467857,"p50_e2el":"10.567636434803717","p50_tpot":0.01113329534606216,"p50_ttft":0.2558715546038002,"mean_e2el":"10.673764683925764","mean_tpot":0.01105389194219896,"mean_ttft":"0.49852078498370245","p50_intvty":89.82066575227428,"mean_intvty":90.46587439329258},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":711822.1617812872,"tput_per_gpu":"1352.4621073844455","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":681.6841628474797,"output_tput_per_gpu":670.7779445369658,"p50_itl":"0.11463270802050829","p90_itl":"0.16742689199745656","mean_itl":"0.11647443206914627","p50_e2el":5.16941646579653,"p50_tpot":0.005593879700062152,"p50_ttft":0.06486689136363566,"p90_e2el":6.801806426560506,"p90_tpot":"0.0072217399779024565","p90_ttft":0.09647683147341009,"mean_e2el":5.334340465228888,"mean_tpot":0.005655500830276864,"mean_ttft":0.1315732661925722,"p50_intvty":"178.76680472568785","p90_intvty":"138.47078447297525","mean_intvty":176.8189997685926},{"ep":1,"hw":"B300","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1104904.0323004439","tput_per_gpu":"2099.3176613708433","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":"1055.9927875846267","output_tput_per_gpu":"1043.3248737862166","p50_itl":"0.13916947389952838","p90_itl":0.222942744800821,"mean_itl":"0.14886877706134116","p50_e2el":6.475469812517986,"p50_tpot":0.006924690412223293,"p50_ttft":0.0722286410164088,"p90_e2el":9.068164726812393,"p90_tpot":0.009528746000091223,"p90_ttft":"0.10938626881688839","mean_e2el":6.807812058206764,"mean_tpot":0.00724685288093104,"mean_ttft":"0.11150847031967714","p50_intvty":"144.41078813210547","p90_intvty":"104.94560354430966","mean_intvty":137.9909343311417},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":740194.2536746678,"tput_per_gpu":"1406.3690819818687","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":"1248.8496655771394","output_tput_per_gpu":"157.51941640472933","p50_itl":0.05283727101050317,"p90_itl":"0.10856676688417792","mean_itl":"0.062427672528723596","p50_e2el":2.474656330421567,"p50_tpot":0.002397240569957772,"p50_ttft":"0.16235008148942143","p90_e2el":4.896762832440436,"p90_tpot":"0.0049074222966131505","p90_ttft":"0.17651558401994408","mean_e2el":"2.9428587093017997","mean_tpot":"0.0029571002063751334","mean_ttft":"0.16794876190833746","p50_intvty":417.1462858304685,"p90_intvty":203.7729666530122,"mean_intvty":"338.16912860921207"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":428109.6155039743,"tput_per_gpu":813.4082694575512,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":722.9204172254273,"output_tput_per_gpu":90.48785223212397,"p50_itl":0.04809412098256871,"p90_itl":0.09043622720055282,"mean_itl":"0.054016249901596855","p50_e2el":2.276520248502493,"p50_tpot":0.002254049335062859,"p50_ttft":"0.15258744049060624","p90_e2el":"3.6684600530978058","p90_tpot":"0.0038894782712053555","p90_ttft":"0.16571968770003875","mean_e2el":"2.5699896609963617","mean_tpot":0.002608158454053474,"mean_ttft":"0.15546338439453394","p50_intvty":443.6460127311778,"p90_intvty":257.1038916461406,"mean_intvty":"383.41228787148583"},{"ep":1,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-07-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":99188.82884218973,"tput_per_gpu":"188.45877480016048","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":93.53882218804371,"output_tput_per_gpu":94.91995261211677,"p50_itl":"0.049176217056810856","p90_itl":0.07028053037356585,"mean_itl":0.05250508546529724,"p50_e2el":"2.3895931504666805","p50_tpot":0.002420631768444077,"p50_ttft":"0.043049433501437306","p90_e2el":"2.9571145204943607","p90_tpot":"0.0028523644061545484","p90_ttft":0.04869518443010751,"mean_e2el":2.465268152562203,"mean_tpot":0.002545078407165981,"mean_ttft":0.08153736494714395,"p50_intvty":413.1152920639291,"p90_intvty":"350.58634087646703","mean_intvty":"392.91520339191794"},{"ep":1,"hw":"B300","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-07-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"160452.37481941478","tput_per_gpu":"304.85951215688806","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":154.1735024786507,"output_tput_per_gpu":"150.68600967823735","p50_itl":0.05659683997510001,"p90_itl":0.08857871805084869,"mean_itl":0.06338392416504969,"p50_e2el":"2.7929705674760044","p50_tpot":0.00293584699420752,"p50_ttft":0.04480569850420579,"p90_e2el":"3.7931623479817067","p90_tpot":0.004237828458911836,"p90_ttft":0.09494320090161705,"mean_e2el":"2.9204061855532926","mean_tpot":"0.0030985762954625364","mean_ttft":0.09665248125093058,"p50_intvty":340.617205860189,"p90_intvty":"235.96990998941328","mean_intvty":322.7288614659482},{"ep":1,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-29","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"3560183.8601421956","tput_per_gpu":6764.349334270171,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":6015.452557919869,"output_tput_per_gpu":748.8967763503027,"p50_itl":"0.16227831796277314","p90_itl":0.3593641599873081,"mean_itl":"0.20491947134527708","p50_e2el":8.774136196501786,"p50_tpot":0.009193325569025544,"p50_ttft":0.2266005139972549,"p90_e2el":"13.305372092901962","p90_tpot":"0.014003329573245482","p90_ttft":0.5437862758000847,"mean_e2el":9.521125786379343,"mean_tpot":0.009960293361912672,"mean_ttft":"0.40092864969265063","p50_intvty":"108.77456612319192","p90_intvty":71.41158784912001,"mean_intvty":"100.39864928315427"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1749201.7552742485","tput_per_gpu":"3323.4833350210724","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":2950.858247363892,"output_tput_per_gpu":"372.62508765718013","p50_itl":0.08259513496886939,"p90_itl":"0.19005437640007583","mean_itl":"0.10197759467321119","p50_e2el":4.418211896496359,"p50_tpot":0.004518077185315774,"p50_ttft":0.1631984959822148,"p90_e2el":6.591734620835633,"p90_tpot":0.00701731280565394,"p90_ttft":0.2747640188084917,"mean_e2el":4.811457141266146,"mean_tpot":0.004967729416356999,"mean_ttft":0.2160920196765801,"p50_intvty":"221.33309347837292","p90_intvty":"142.50469199467452","mean_intvty":201.2992085896122},{"ep":2,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":4910480.753864552,"tput_per_gpu":9329.913432342648,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":4706.657872110482,"output_tput_per_gpu":4623.255560232166,"p50_itl":0.7283334691310301,"p90_itl":"1.1271875431295484","mean_itl":0.8278899790975537,"p50_e2el":45.35593241953757,"p50_tpot":0.03568645857700628,"p50_ttft":"11.642667262465693","p90_e2el":73.01680200365377,"p90_tpot":"0.054856601034026835","p90_ttft":"16.143146195751616","mean_e2el":49.56522482485243,"mean_tpot":"0.040307240366554184","mean_ttft":12.46863915251688,"p50_intvty":"28.021833487403715","p90_intvty":"18.229346717630445","mean_intvty":"24.809438475717926"},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":6143168.28049042,"tput_per_gpu":"11672.019732931798","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":"10381.279847214668","output_tput_per_gpu":1290.739885717131,"p50_itl":"0.41106357891112566","p90_itl":0.8229213299229743,"mean_itl":"0.47657566099239007","p50_e2el":"20.530007042572834","p50_tpot":0.02154608888655836,"p50_ttft":"0.27938759641256183","p90_e2el":31.75894493267406,"p90_tpot":0.03380267301433347,"p90_ttft":0.8988701118854806,"mean_e2el":"22.225353001760958","mean_tpot":0.02321105070700239,"mean_ttft":0.8650830264559772,"p50_intvty":46.41213564397088,"p90_intvty":29.58345926004036,"mean_intvty":"43.082926862001834"},{"ep":4,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-30","num_gpus":16,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1239114.6898542903","tput_per_gpu":1208.136822607933,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":"1074.3052085674065","output_tput_per_gpu":"133.83161404052655","p50_itl":0.8999792760005221,"p90_itl":3.412066083203536,"mean_itl":"1.4805348748226748","p50_e2el":26.63240777651663,"p50_tpot":"0.026606008396318822","p50_ttft":2.539398918976076,"p90_e2el":31.63123974430491,"p90_tpot":"0.029170481281710643","p90_ttft":4.739969738497165,"mean_e2el":"26.793499694599085","mean_tpot":"0.025802194617830586","mean_ttft":3.169715225237087,"p50_intvty":"37.585495167263005","p90_intvty":"34.281230753192325","mean_intvty":38.75639319877662},{"ep":1,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-07-29","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"2538455.3998045065","tput_per_gpu":4823.065259628563,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":4282.308789990962,"output_tput_per_gpu":540.7564696376004,"p50_itl":0.1092534251511097,"p90_itl":0.2585815296508372,"mean_itl":"0.14066313250269172","p50_e2el":5.965767402434722,"p50_tpot":0.006306823992613921,"p50_ttft":0.2220692418050021,"p90_e2el":9.70255262246356,"p90_tpot":0.009954677712265962,"p90_ttft":0.3876936782151461,"mean_e2el":"6.6101777340692935","mean_tpot":0.006810064170288237,"mean_ttft":0.2908868057478685,"p50_intvty":158.5584124705438,"p90_intvty":"100.45528633918698","mean_intvty":"146.84149444037837"},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"5983439.9840871645","tput_per_gpu":"11368.535969765613","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":"10097.593603566924","output_tput_per_gpu":"1270.9423661986887","p50_itl":0.6497619119472802,"p90_itl":"1.9614596794825048","mean_itl":0.906810098722285,"p50_e2el":11.50165023712907,"p50_tpot":"0.011344358171676412","p50_ttft":"1.0055775489890948","p90_e2el":"13.173358069499955","p90_tpot":"0.012597778794952096","p90_ttft":1.795001518027857,"mean_e2el":"11.372710639786238","mean_tpot":"0.011020434040155042","mean_ttft":"1.1795985977121746","p50_intvty":88.14954401710547,"p90_intvty":79.37907279342753,"mean_intvty":90.74052767398365},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"7780171.9026986435","tput_per_gpu":"14782.326615127422","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":"13142.827913664314","output_tput_per_gpu":"1639.4987014631088","p50_itl":0.9765073340386152,"p90_itl":"3.0602209202479567","mean_itl":"1.3914886670075188","p50_e2el":"17.507051842520013","p50_tpot":0.01754322197247265,"p50_ttft":1.540470527135767,"p90_e2el":"22.080996382329612","p90_tpot":"0.020528780141246816","p90_ttft":"2.5612190366489815","mean_e2el":"17.680461462909808","mean_tpot":0.01719841022236122,"mean_ttft":"1.8327102664239647","p50_intvty":57.00207188674442,"p90_intvty":48.71210043263998,"mean_intvty":58.14490915560374},{"ep":1,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":64,"date":"2026-07-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"1552388.8335013145","tput_per_gpu":"2949.5387836524974","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":1491.634519131026,"output_tput_per_gpu":"1457.9042645214713","p50_itl":"0.25852380879223347","p90_itl":0.4051664303056895,"mean_itl":"0.43235715432845745","p50_e2el":"12.106127392034978","p50_tpot":"0.012856437185942083","p50_ttft":"0.11709394911304116","p90_e2el":"50.344455534033486","p90_tpot":"0.053727069334458165","p90_ttft":0.6807086063083275,"mean_e2el":19.68709491844129,"mean_tpot":"0.021033169056184707","mean_ttft":"0.39113031239248813","p50_intvty":77.78204688725532,"p90_intvty":18.61259161140666,"mean_intvty":47.54395295015967},{"ep":1,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-07-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":5449527.702617769,"tput_per_gpu":"10354.102634973762","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":5223.330208681553,"output_tput_per_gpu":5130.772426292209,"p50_itl":0.864478763891384,"p90_itl":1.048216071119532,"mean_itl":0.9409920207772033,"p50_e2el":"41.393743097549304","p50_tpot":"0.042958813764751524","p50_ttft":0.6442798324860632,"p90_e2el":69.00696800621301,"p90_tpot":0.05488275060367714,"p90_ttft":4.576707075443119,"mean_e2el":"44.957747165493494","mean_tpot":"0.046820256095879954","mean_ttft":"1.8338473005447669","p50_intvty":"23.278110179581304","p90_intvty":18.22066111848629,"mean_intvty":"21.358277023350094"},{"ep":1,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-29","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":4668772.321894686,"tput_per_gpu":8870.667411599903,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":7882.703706331737,"output_tput_per_gpu":987.9637052681664,"p50_itl":0.2556639130052645,"p90_itl":0.5295875535084633,"mean_itl":"0.31286610910987583","p50_e2el":"13.868904102491797","p50_tpot":"0.014499329648469254","p50_ttft":0.2337206204974791,"p90_e2el":19.7449526360113,"p90_tpot":"0.020621055574829822","p90_ttft":0.5528400760929801,"mean_e2el":"14.647113853413249","mean_tpot":"0.015301158526786415","mean_ttft":0.5231646680907034,"p50_intvty":68.96870574327369,"p90_intvty":"48.494122736403746","mean_intvty":65.35452843321546},{"ep":1,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":2,"date":"2026-07-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":283841.603855496,"tput_per_gpu":539.2990473254424,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":267.673382401258,"output_tput_per_gpu":"271.62566492418443","p50_itl":0.0684900286141783,"p90_itl":0.100954413972795,"mean_itl":0.07272446411303603,"p50_e2el":3.215542693156749,"p50_tpot":0.003302929134997552,"p50_ttft":0.06369229895062745,"p90_e2el":"4.0902569402474915","p90_tpot":"0.0044530717103484175","p90_ttft":0.08834823193028601,"mean_e2el":"3.4211774900788443","mean_tpot":0.003533233683143394,"mean_ttft":"0.11221449044533074","p50_intvty":"302.76156681779406","p90_intvty":224.5640908220986,"mean_intvty":283.026850097935},{"ep":2,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"4167138.9024930927","tput_per_gpu":"7917.5639147368765","model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":"3994.4872556798355","output_tput_per_gpu":3923.076659057041,"p50_itl":0.5304610184393823,"p90_itl":0.8251452775672078,"mean_itl":0.6377348783322311,"p50_e2el":"24.660210461588576","p50_tpot":"0.026432227401288658","p50_ttft":"0.24642446427606046","p90_e2el":47.44592799926108,"p90_tpot":0.04470975048646684,"p90_ttft":0.6022770617622882,"mean_e2el":"29.192920895751012","mean_tpot":"0.031018001037246733","mean_ttft":0.6273807255793145,"p50_intvty":37.83260429846509,"p90_intvty":22.366485813932,"mean_intvty":"32.239343818423045"},{"ep":1,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-07-30","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"319614.98865299847","tput_per_gpu":311.6246139366735,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":"276.82091412211986","output_tput_per_gpu":"34.803699814553624","p50_itl":"0.26549831047304906","p90_itl":0.8216839288827036,"mean_itl":"0.37058836372813825","p50_e2el":6.676512764504878,"p50_tpot":0.006511803247464986,"p50_ttft":0.8304922664829064,"p90_e2el":"7.4233098641736435","p90_tpot":0.006688677749304411,"p90_ttft":0.8850559312733821,"mean_e2el":6.682145532593131,"mean_tpot":0.00630005852759963,"mean_ttft":0.826849592779763,"p50_intvty":153.5672934204969,"p90_intvty":"149.50638040590235","mean_intvty":158.7286841255755},{"ep":1,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-30","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":748306.9205077683,"tput_per_gpu":729.599247495074,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":648.4594231610249,"output_tput_per_gpu":81.13982433404905,"p50_itl":0.3840415980084799,"p90_itl":"1.4475693690474145","mean_itl":0.6108011500877736,"p50_e2el":"10.824106211017352","p50_tpot":"0.010919970317921222","p50_ttft":0.9744464899995364,"p90_e2el":"12.690710764925463","p90_tpot":"0.012251778855051123","p90_ttft":1.800128143298208,"mean_e2el":"11.136933345872967","mean_tpot":"0.010830155757990411","mean_ttft":"1.1961447821202456","p50_intvty":91.57534048960353,"p90_intvty":81.620800687871,"mean_intvty":92.33477544976277},{"ep":4,"hw":"RTX PRO 6000","tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-07-30","num_gpus":16,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":1773642.227421315,"tput_per_gpu":1729.301171735782,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"serving_config":"agg","input_tput_per_gpu":1537.505448415877,"output_tput_per_gpu":191.795723319905,"p50_itl":"2.1665497754584067","p90_itl":9.992794622929067,"mean_itl":4.078385031871001,"p50_e2el":76.37623386702035,"p50_tpot":0.07694602480140297,"p50_ttft":5.814127494988497,"p90_e2el":88.49308526854148,"p90_tpot":0.08504243798882256,"p90_ttft":"14.817927211982912","mean_e2el":76.01638929022901,"mean_tpot":0.07354872872681747,"mean_ttft":8.26421806213475,"p50_intvty":"12.996122965169304","p90_intvty":"11.758835043410134","mean_intvty":"13.596428073071209"},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":4507079.352545787,"tput_per_gpu":8563.450769836994,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":7609.703079850748,"output_tput_per_gpu":953.7476899862471,"p50_itl":"0.27649610210210085","p90_itl":0.5820451005827636,"mean_itl":"0.32143695321290705","p50_e2el":"13.058894061483443","p50_tpot":"0.013619486086688865","p50_ttft":"0.25247951596975327","p90_e2el":23.30217289165594,"p90_tpot":"0.024319216876805032","p90_ttft":0.5782461978495123,"mean_e2el":"14.842685326433275","mean_tpot":0.0154742677261209,"mean_ttft":0.5497903852869058,"p50_intvty":73.42420952119181,"p90_intvty":"41.119745140879566","mean_intvty":64.62341337884301},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":128,"date":"2026-07-29","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":7378817.569681069,"tput_per_gpu":14019.75338239403,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785308400,"serving_config":"agg","input_tput_per_gpu":"12468.609006066672","output_tput_per_gpu":"1551.1443763273585","p50_itl":0.7427383398171514,"p90_itl":"1.3012008849531411","mean_itl":0.7989810666736145,"p50_e2el":"35.525833779596724","p50_tpot":0.03713679848666402,"p50_ttft":0.4397349974606186,"p90_e2el":"50.221949814446276","p90_tpot":0.05228833006669162,"p90_ttft":1.054814881156204,"mean_e2el":"37.306268884619385","mean_tpot":0.03900102437755125,"mean_ttft":1.47134402179272,"p50_intvty":"26.927469269035246","p90_intvty":"19.124726276867918","mean_intvty":"25.640352169200817"},{"ep":1,"hw":"B300","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":1316939.027427228,"tput_per_gpu":2502.184152111733,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":2223.912507437157,"output_tput_per_gpu":278.2716446745763,"p50_itl":"0.18994120345450938","p90_itl":0.5788378432393075,"mean_itl":"0.26655206817193844","p50_e2el":"3.2623535091988742","p50_tpot":0.003259661574326042,"p50_ttft":"0.25777142820879817","p90_e2el":3.599390026880428,"p90_tpot":"0.0035313095595053347","p90_ttft":"0.41880719484761375","mean_e2el":3.242744727875106,"mean_tpot":0.003203763532215341,"mean_ttft":"0.29734460460022094","p50_intvty":306.7803135995052,"p90_intvty":"283.18106446042634","mean_intvty":312.1328993056236},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":3104504.986548876,"tput_per_gpu":"5898.5594744428645","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":5235.642000366254,"output_tput_per_gpu":662.9174740766103,"p50_itl":"0.30704615550348535","p90_itl":0.9860939435893672,"mean_itl":"0.44509776195866935","p50_e2el":5.419695902994135,"p50_tpot":0.005554958880083385,"p50_ttft":0.3416801800194662,"p90_e2el":6.570330919406842,"p90_tpot":0.006167774099384952,"p90_ttft":0.5786744616983966,"mean_e2el":"5.4535257652263684","mean_tpot":0.005399809199796032,"mean_ttft":0.4461761527993076,"p50_intvty":"180.01933436183944","p90_intvty":"162.13304571250748","mean_intvty":"185.19172863325858"},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":4,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":2169425.111264675,"tput_per_gpu":4121.907711402882,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":3663.504185395065,"output_tput_per_gpu":458.4035260078172,"p50_itl":"0.22420896007679403","p90_itl":0.7251087985583582,"mean_itl":0.3259542044416197,"p50_e2el":"3.9816020934958942","p50_tpot":0.00393786620364088,"p50_ttft":0.3025043049710803,"p90_e2el":4.369876018085051,"p90_tpot":0.004250168081052215,"p90_ttft":0.5276755547500216,"mean_e2el":"3.9243017481203424","mean_tpot":0.003906111006764486,"mean_ttft":0.338805808997131,"p50_intvty":"253.94463607611095","p90_intvty":"235.28481248968154","mean_intvty":"256.00910938481525"},{"ep":2,"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-07-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"3474342.3662061556","tput_per_gpu":6601.250495791695,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1783321200,"serving_config":"agg","input_tput_per_gpu":3337.646694432628,"output_tput_per_gpu":"3263.6038013590673","p50_itl":0.3580663129687309,"p90_itl":0.5622624469688162,"mean_itl":0.380995606831815,"p50_e2el":16.48515059403144,"p50_tpot":"0.017591891242135865","p50_ttft":0.1668793399585411,"p90_e2el":23.25037338198163,"p90_tpot":"0.024762144923547467","p90_ttft":"0.28298792454879734","mean_e2el":17.39571664303603,"mean_tpot":"0.018537505228184835","mean_ttft":"0.37414267829135495","p50_intvty":"56.844371434312485","p90_intvty":40.38422370467002,"mean_intvty":53.94469146147982},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":128,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":9673880.998173172,"tput_per_gpu":"18380.373896529025","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":"16345.178562990255","output_tput_per_gpu":2035.19533353877,"p50_itl":"1.5145233978983015","p90_itl":5.044433552259579,"mean_itl":"2.2670691192902264","p50_e2el":28.72531498398166,"p50_tpot":0.02899949348917332,"p50_ttft":2.146304373978637,"p90_e2el":33.74718070791569,"p90_tpot":0.03227014827477755,"p90_ttft":4.264459088689182,"mean_e2el":28.5222007547879,"mean_tpot":"0.027940673487172347","mean_ttft":"2.8281604756869454","p50_intvty":34.48336090329786,"p90_intvty":30.98839185630898,"mean_intvty":"35.790117960438685"},{"ep":2,"hw":"B300","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-08-10","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":4390622.076466195,"tput_per_gpu":8342.18194528577,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1786345200,"serving_config":"agg","input_tput_per_gpu":7418.074962148443,"output_tput_per_gpu":924.106983137328,"p50_itl":"0.46327176853083074","p90_itl":1.348129225242883,"mean_itl":0.6236730146727376,"p50_e2el":7.777872251113877,"p50_tpot":0.007773719703436179,"p50_ttft":0.6845782310701907,"p90_e2el":8.958465047366916,"p90_tpot":0.008620257274063622,"p90_ttft":"1.4753814581781626","mean_e2el":"7.7176628048182465","mean_tpot":0.007553921505308664,"mean_ttft":0.8026886011386523,"p50_intvty":128.6385460435337,"p90_intvty":"116.00581841202938","mean_intvty":132.3815715184796},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":48,"date":"2026-07-30","num_gpus":18,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"2035862.6274417609","tput_per_gpu":4316.028770176533,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":34543.83751464092,"output_tput_per_gpu":537.5526771184839,"p50_itl":1.088907803496113,"mean_itl":0.9433819225090256,"p50_e2el":4.902269474521745,"p50_tpot":"0.0038067211377980762","p50_ttft":1.37002596797538,"mean_e2el":5.016091160191611,"mean_tpot":"0.0037773837832486418","mean_ttft":"1.5447406147920144","p50_intvty":"262.69326378302316","mean_intvty":264.7334921155339},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":1536,"date":"2026-07-30","num_gpus":34,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":10016013.78002487,"tput_per_gpu":"21233.949213652722","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":18,"input_tput_per_gpu":35649.03812920835,"output_tput_per_gpu":"5016.9741836526455","p50_itl":3.165153119014576,"mean_itl":2.726200770156057,"p50_e2el":17.010709334485,"p50_tpot":"0.011037713067178238","p50_ttft":7.075090130005265,"mean_e2el":"17.126151157264434","mean_tpot":0.0108892808902315,"mean_ttft":7.093807871189725,"p50_intvty":90.59847759347919,"mean_intvty":91.83342867912195},{"ep":1,"hw":"GB300 NVL72","tp":4,"isl":8192,"osl":1024,"conc":90,"date":"2026-07-30","num_gpus":64,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":"1175319.5641411804","tput_per_gpu":"2491.6774759793025","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":60,"serving_config":"disagg","num_prefill_gpu":4,"input_tput_per_gpu":"35450.407208771416","output_tput_per_gpu":"294.42882712649504","p50_itl":0.912371993996203,"mean_itl":0.7866333039762212,"p50_e2el":4.436624693014892,"p50_tpot":0.00316835006170215,"p50_ttft":"1.5241092669893987","mean_e2el":4.564161975134387,"mean_tpot":"0.0031390024828987523","mean_ttft":"1.6792937172507663","p50_intvty":315.6216896887854,"mean_intvty":318.5725418976213},{"ep":16,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":384,"date":"2026-07-30","num_gpus":24,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":6281898.550334637,"tput_per_gpu":13317.62492670943,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":16,"serving_config":"disagg","num_prefill_gpu":8,"input_tput_per_gpu":"35515.848080925716","output_tput_per_gpu":"2218.5133496012877","p50_itl":"1.7410581090371124","mean_itl":"1.4992524266369904","p50_e2el":9.54508664298919,"p50_tpot":"0.0060704183108611235","p50_ttft":"4.0280418315087445","mean_e2el":9.628780198615374,"mean_tpot":0.006004519969378577,"mean_ttft":4.100531001049613,"p50_intvty":"164.73329329064708","mean_intvty":"166.54120647441073"},{"ep":1,"hw":"GB300 NVL72","tp":16,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-30","num_gpus":66,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":95928.17036923488,"tput_per_gpu":"203.36772118277793","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":64,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":5955.664808846439,"output_tput_per_gpu":23.60843719328852,"p50_itl":0.6508948060218245,"mean_itl":0.5685635874686942,"p50_e2el":2.393138960993383,"p50_tpot":0.002298864351551767,"p50_ttft":"0.24254019247018732","mean_e2el":"2.4065826944197397","mean_tpot":0.002290829980723522,"mean_ttft":0.2833532134691268,"p50_intvty":"434.99739309323985","mean_intvty":436.523010618259},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":480,"date":"2026-07-30","num_gpus":52,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":4346029.913455932,"tput_per_gpu":9213.583416526577,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":40,"serving_config":"disagg","num_prefill_gpu":12,"input_tput_per_gpu":35491.24361684043,"output_tput_per_gpu":1330.285356432421,"p50_itl":"1.5220715310133528","mean_itl":1.308186191322104,"p50_e2el":7.911079261015402,"p50_tpot":0.005279281359805776,"p50_ttft":3.103454393480206,"mean_e2el":8.036960965528595,"mean_tpot":0.005210250506046247,"mean_ttft":"3.2395970265063623","p50_intvty":"189.41972057287546","mean_intvty":191.9293513506784},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":42,"date":"2026-07-30","num_gpus":58,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":641362.3011632989,"tput_per_gpu":"1359.6880784661937","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":2,"input_tput_per_gpu":"35061.583374406764","output_tput_per_gpu":"156.04896075403036","p50_itl":0.8302873549982905,"mean_itl":0.7177594403548089,"p50_e2el":4.192702066997299,"p50_tpot":"0.0029000403765201323","p50_ttft":"1.5296127694891766","mean_e2el":4.282353487289496,"mean_tpot":"0.0028713664845213377","mean_ttft":1.640956111580159,"p50_intvty":344.8227852606444,"mean_intvty":348.2662367868036},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":192,"date":"2026-07-30","num_gpus":38,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":2981485.744046492,"tput_per_gpu":6320.749777378563,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":32,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":35582.16067506442,"output_tput_per_gpu":834.2352340624643,"p50_itl":"1.2113214879937004","mean_itl":1.041557405305918,"p50_e2el":6.364123884006403,"p50_tpot":"0.0041972869374184325","p50_ttft":2.522036935988581,"mean_e2el":6.43886727071731,"mean_tpot":0.004147803057697228,"mean_ttft":"2.6198235484087036","p50_intvty":"238.24913924399368","mean_intvty":"241.09148532119042"},{"ep":8,"hw":"GB300 NVL72","tp":8,"isl":8192,"osl":1024,"conc":168,"date":"2026-07-30","num_gpus":62,"sequence":"8K/1K","framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":1812114.925356014,"tput_per_gpu":"3841.6836417547493","model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"single_turn","date_timestamp":1785394800,"num_decode_gpu":56,"serving_config":"disagg","num_prefill_gpu":6,"input_tput_per_gpu":35283.8084815362,"output_tput_per_gpu":"472.88455177816564","p50_itl":1.051945358980447,"mean_itl":0.903973861021668,"p50_e2el":5.571591001993511,"p50_tpot":"0.0036537203455301384","p50_ttft":2.223109583981568,"mean_e2el":5.635010662200697,"mean_tpot":"0.0036116603886394176","mean_ttft":"2.3079968478374604","p50_intvty":273.693634277011,"mean_intvty":276.8809612181502},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":32,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":44496.95056684492,"tput_per_gpu":83.20929756,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":143.1101732},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":14,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":10761.5535828877,"tput_per_gpu":20.1241052,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":266.7910169},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":128,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":67453.43732620322,"tput_per_gpu":126.1379278,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":126.296975},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":128,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":67099.05660377358,"tput_per_gpu":142.25,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":137.1742},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":256,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":81037.09930481283,"tput_per_gpu":151.5393757,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":94.08651502},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":28,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":31952.15151871658,"tput_per_gpu":59.75052334,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":175.3054507},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":256,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":97429.59620320857,"tput_per_gpu":182.1933449,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":61.82939599},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":12,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"11768.867924528302","tput_per_gpu":24.95,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":275.4821},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"32276833.058479536","tput_per_gpu":55193.38453,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"192.6989","input_tput_per_gpu":49082.91,"kv_cache_precision":"FP8","total_input_tokens":9458221,"output_tput_per_gpu":6110.477,"total_output_tokens":1177482,"tput_requests_per_sec":"6.642488","p50_e2el":20.67846,"p50_tpot":0.020028,"p50_ttft":0.190048,"p50_intvty":49.93066},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"37713538.169590645","tput_per_gpu":64490.15027,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"329.4641","input_tput_per_gpu":57325.18,"kv_cache_precision":"FP8","total_input_tokens":18886588,"output_tput_per_gpu":7164.975,"total_output_tokens":2360602,"tput_requests_per_sec":"7.770194","p50_e2el":35.28142,"p50_tpot":0.034229,"p50_ttft":0.26506,"p50_intvty":29.21491},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"26465229.292397663","tput_per_gpu":45255.54209,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"117.6293","input_tput_per_gpu":40241.05,"kv_cache_precision":"FP8","total_input_tokens":4733525,"output_tput_per_gpu":5014.492,"total_output_tokens":589851,"tput_requests_per_sec":"5.440823","p50_e2el":12.62134,"p50_tpot":0.012163,"p50_ttft":0.178562,"p50_intvty":82.21638},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":1024,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"124877.35849056605","tput_per_gpu":264.74,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":26.4971},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":512,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"113825.47169811321","tput_per_gpu":241.31,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":44.0723},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":128,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":69405.6603773585,"tput_per_gpu":147.14,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":132.626},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":256,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":99683.96226415095,"tput_per_gpu":211.33,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":70.8717},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":28,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"20627.358490566035","tput_per_gpu":43.73,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":226.7574},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":1024,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":124231.1320754717,"tput_per_gpu":263.37,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":26.7666},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":20099699.56140351,"tput_per_gpu":34370.48625,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"78.10109","input_tput_per_gpu":30303.83,"kv_cache_precision":"FP8","total_input_tokens":4733525,"output_tput_per_gpu":3776.202,"total_output_tokens":589851,"tput_requests_per_sec":"8.194507","p50_e2el":8.271427,"p50_tpot":0.007989,"p50_ttft":0.098709,"p50_intvty":124.8632},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":256,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":22605032.59649123,"tput_per_gpu":38654.60574,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"122.514","input_tput_per_gpu":19375.64,"kv_cache_precision":"FP8","total_input_tokens":2373787,"output_tput_per_gpu":19278.96,"total_output_tokens":2361942,"tput_requests_per_sec":"20.89558","p50_e2el":12.93751,"p50_tpot":0.012546,"p50_ttft":0.102974,"p50_intvty":79.70679},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":384,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":25475204.43859649,"tput_per_gpu":43562.59959,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"163.0554","input_tput_per_gpu":21852.86,"kv_cache_precision":"FP8","total_input_tokens":3563226,"output_tput_per_gpu":21709.74,"total_output_tokens":3539890,"tput_requests_per_sec":"23.55028","p50_e2el":17.08881,"p50_tpot":0.016566,"p50_ttft":0.142053,"p50_intvty":60.36553},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":512,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"28498719.292397663","tput_per_gpu":48732.80999,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"194.4037","input_tput_per_gpu":24448.36,"kv_cache_precision":"FP8","total_input_tokens":4752851,"output_tput_per_gpu":24284.45,"total_output_tokens":4720987,"tput_requests_per_sec":"26.33695","p50_e2el":20.55348,"p50_tpot":0.019942,"p50_ttft":0.15266,"p50_intvty":50.14503},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":4,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"1456150.0894736843","tput_per_gpu":2490.016653,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"18.57066","input_tput_per_gpu":1251.396,"kv_cache_precision":"FP8","total_input_tokens":92957,"output_tput_per_gpu":1238.621,"total_output_tokens":92008,"tput_requests_per_sec":"5.384839","p50_e2el":1.916041,"p50_tpot":0.001836,"p50_ttft":0.037932,"p50_intvty":544.697},{"hw":"GB200 NVL72","isl":131072,"osl":8192,"conc":20,"date":"2025-12-15","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"20156.265240641715","tput_per_gpu":37.692216,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1765785600,"serving_config":"agg","p50_intvty":235.9325659},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":512,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":95424.5283018868,"tput_per_gpu":202.3,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":85.6898},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":56,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":50683.96226415095,"tput_per_gpu":107.45,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":173.3102},{"hw":"GB300 NVL72","isl":131072,"osl":8192,"conc":256,"date":"2025-10-31","sequence":"128K/8K","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":91094.33962264152,"tput_per_gpu":193.12,"model_display":"DeepSeek-R1","spec_decoding":"mtp","date_timestamp":1761894000,"serving_config":"agg","p50_intvty":105.3741},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":250793.8269343066,"dataset_name":"random","tput_per_gpu":343.5875429,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"16.76428648","input_tput_per_gpu":305.4111493,"total_input_tokens":40960,"output_tput_per_gpu":38.17639366,"total_output_tokens":5120,"tput_requests_per_sec":"0.298253075","p50_e2el":3.353059788,"p50_tpot":0.003196648,"p50_ttft":0.082888682,"p50_intvty":312.8276677},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":471869.4991240876,"dataset_name":"random","tput_per_gpu":646.4612138,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"17.82009462","input_tput_per_gpu":574.63219,"total_input_tokens":81920,"output_tput_per_gpu":71.82902376,"total_output_tokens":10240,"tput_requests_per_sec":"0.561164248","p50_e2el":3.560005599,"p50_tpot":0.003373031,"p50_ttft":0.109395226,"p50_intvty":296.4692879},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1536,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":37095611.1754386,"tput_per_gpu":63433.49511,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"448.1003","input_tput_per_gpu":31826.71,"kv_cache_precision":"FP8","total_input_tokens":14261558,"output_tput_per_gpu":31606.78,"total_output_tokens":14163009,"tput_requests_per_sec":"34.27804","p50_e2el":46.54151,"p50_tpot":0.041247,"p50_ttft":4.345391,"p50_intvty":24.24393},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":12370212.81871345,"tput_per_gpu":21153.06392,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"55.94816","input_tput_per_gpu":10628.45,"kv_cache_precision":"FP8","total_input_tokens":1189284,"output_tput_per_gpu":10524.62,"total_output_tokens":1177666,"tput_requests_per_sec":"22.87832","p50_e2el":5.81823,"p50_tpot":0.005629,"p50_ttft":0.059314,"p50_intvty":177.6376},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":640,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"30405119.023391817","tput_per_gpu":51992.75353,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"227.7071","input_tput_per_gpu":26090.0,"kv_cache_precision":"FP8","total_input_tokens":5940879,"output_tput_per_gpu":25902.75,"total_output_tokens":5898240,"tput_requests_per_sec":"28.10628","p50_e2el":24.15853,"p50_tpot":0.023439,"p50_ttft":0.180157,"p50_intvty":42.66344},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1024,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"35688604.385964915","tput_per_gpu":61027.5135,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"310.4982","input_tput_per_gpu":30622.66,"kv_cache_precision":"FP8","total_input_tokens":9508283,"output_tput_per_gpu":30404.85,"total_output_tokens":9440652,"tput_requests_per_sec":"32.97925","p50_e2el":33.33883,"p50_tpot":0.032394,"p50_ttft":0.199619,"p50_intvty":30.86977},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":16,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":"10393234.701754386","tput_per_gpu":17772.43134,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"37.17187","input_tput_per_gpu":15804.16,"kv_cache_precision":"FP8","total_input_tokens":1174940,"output_tput_per_gpu":1968.276,"total_output_tokens":146329,"tput_requests_per_sec":"4.30433","p50_e2el":3.962087,"p50_tpot":0.003797,"p50_ttft":0.077889,"p50_intvty":263.3748},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":32,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":5567275.549122807,"tput_per_gpu":9520.041189,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"31.11536","input_tput_per_gpu":4762.166,"kv_cache_precision":"FP8","total_input_tokens":296353,"output_tput_per_gpu":4757.875,"total_output_tokens":296086,"tput_requests_per_sec":"10.28431","p50_e2el":3.213565,"p50_tpot":0.003106,"p50_ttft":0.036339,"p50_intvty":321.979},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":16,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":3530984.303508772,"tput_per_gpu":6037.983159,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"24.43324","input_tput_per_gpu":3043.354,"kv_cache_precision":"FP8","total_input_tokens":148718,"output_tput_per_gpu":2994.629,"total_output_tokens":146337,"tput_requests_per_sec":"6.548456","p50_e2el":2.610728,"p50_tpot":0.002515,"p50_ttft":0.037376,"p50_intvty":397.536},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":7746494.671532848,"dataset_name":"random","tput_per_gpu":10612.6977,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"69.47149731","input_tput_per_gpu":9433.50907,"total_input_tokens":2621440,"output_tput_per_gpu":1179.188634,"total_output_tokens":327680,"tput_requests_per_sec":"4.606205601","p50_e2el":14.06502454,"p50_tpot":0.013194243,"p50_ttft":0.567314194,"p50_intvty":75.79063218},{"dp":1,"hw":"H200","pp":1,"tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":9888451.503649635,"dataset_name":"random","tput_per_gpu":13547.17856,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"108.8462807","input_tput_per_gpu":12041.9365,"total_input_tokens":2621440,"output_tput_per_gpu":1505.242063,"total_output_tokens":327680,"tput_requests_per_sec":"2.939925904","p50_e2el":21.64914323,"p50_tpot":0.020381064,"p50_ttft":0.799314681,"p50_intvty":49.06515167},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":32,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":5813762.909489051,"dataset_name":"random","tput_per_gpu":7964.855186,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"46.28332737","input_tput_per_gpu":7079.871276,"total_input_tokens":1310720,"output_tput_per_gpu":884.9839095,"total_output_tokens":163840,"tput_requests_per_sec":"3.456968397","p50_e2el":9.233065663,"p50_tpot":0.008478001,"p50_ttft":0.560071198,"p50_intvty":117.9523409},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"3753861.2408759124","dataset_name":"random","tput_per_gpu":5142.7899,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"35.84046862","input_tput_per_gpu":4571.3688,"total_input_tokens":655360,"output_tput_per_gpu":571.4211,"total_output_tokens":81920,"tput_requests_per_sec":"2.232113672","p50_e2el":7.171786936,"p50_tpot":0.006475682,"p50_ttft":0.547163825,"p50_intvty":154.4238794},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":768,"date":"2026-02-05","num_gpus":1,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":32930397.76023392,"tput_per_gpu":56310.98017,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"252.3166","input_tput_per_gpu":28252.88,"kv_cache_precision":"FP8","total_input_tokens":7128671,"output_tput_per_gpu":28058.1,"total_output_tokens":7079524,"tput_requests_per_sec":"30.43795","p50_e2el":27.14342,"p50_tpot":0.026373,"p50_ttft":0.163626,"p50_intvty":37.91727},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"2869710.7058479534","tput_per_gpu":4907.205307,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"17.62566","input_tput_per_gpu":4204.834,"kv_cache_precision":"FP8","total_input_tokens":74113,"output_tput_per_gpu":527.8099,"total_output_tokens":9303,"tput_requests_per_sec":"0.567354","p50_e2el":1.631976,"p50_tpot":0.001501,"p50_ttft":0.096344,"p50_intvty":616.7399},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":8,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":7020539.50877193,"tput_per_gpu":12005.12256,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"27.48818","input_tput_per_gpu":10657.83,"kv_cache_precision":"FP8","total_input_tokens":585929,"output_tput_per_gpu":1347.288,"total_output_tokens":74069,"tput_requests_per_sec":"2.910342","p50_e2el":2.902755,"p50_tpot":0.002764,"p50_ttft":0.075179,"p50_intvty":361.7941},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"19929844.795321636","tput_per_gpu":34080.0346,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"78.10109","input_tput_per_gpu":30303.83,"kv_cache_precision":"FP8","total_input_tokens":4733525,"output_tput_per_gpu":3776.202,"total_output_tokens":589851,"tput_requests_per_sec":"8.194507","p50_e2el":8.271427,"p50_tpot":0.007989,"p50_ttft":0.098709,"p50_intvty":125.1726},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":64,"date":"2026-02-05","num_gpus":2,"sequence":"1K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":8304329.47368421,"tput_per_gpu":14200.4034,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"41.70322","input_tput_per_gpu":7130.409,"kv_cache_precision":"FP8","total_input_tokens":594722,"output_tput_per_gpu":7069.994,"total_output_tokens":589683,"tput_requests_per_sec":"15.34654","p50_e2el":4.206697,"p50_tpot":0.004071,"p50_ttft":0.041819,"p50_intvty":245.6255},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":8068229.777777779,"tput_per_gpu":13796.67292,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"29.89572","input_tput_per_gpu":12263.16,"kv_cache_precision":"FP8","total_input_tokens":733232,"output_tput_per_gpu":1533.514,"total_output_tokens":91691,"tput_requests_per_sec":"3.34496","p50_e2el":3.206322,"p50_tpot":0.003058,"p50_ttft":0.077774,"p50_intvty":326.9887},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":512,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"15073755.226277374","dataset_name":"random","tput_per_gpu":20651.04466,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"1142.458427","input_tput_per_gpu":18356.48414,"total_input_tokens":20971520,"output_tput_per_gpu":2294.560518,"total_output_tokens":2621440,"tput_requests_per_sec":"2.240781756","p50_e2el":236.8557691,"p50_tpot":0.078611908,"p50_ttft":156.4357867,"p50_intvty":12.72071904},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":14974979.80291971,"dataset_name":"random","tput_per_gpu":20515.72233,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"574.9970589","input_tput_per_gpu":18236.19762,"total_input_tokens":10485760,"output_tput_per_gpu":2279.524703,"total_output_tokens":1310720,"tput_requests_per_sec":"2.226098343","p50_e2el":99.04666932,"p50_tpot":0.077595425,"p50_ttft":19.6665494,"p50_intvty":12.8873577},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":64,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":12396462.81751825,"dataset_name":"random","tput_per_gpu":16983.15406,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"173.6497231","input_tput_per_gpu":15096.13694,"total_input_tokens":2621440,"output_tput_per_gpu":1887.017118,"total_output_tokens":327680,"tput_requests_per_sec":"1.842790154","p50_e2el":34.66476974,"p50_tpot":0.032848843,"p50_ttft":1.060403298,"p50_intvty":30.44247246},{"dp":1,"hw":"H200","pp":1,"tp":1,"isl":8192,"osl":1024,"conc":32,"date":"2026-03-27","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":9721311.693430657,"dataset_name":"random","tput_per_gpu":13318.19702,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"110.7176894","input_tput_per_gpu":11838.39735,"total_input_tokens":1310720,"output_tput_per_gpu":1479.799668,"total_output_tokens":163840,"tput_requests_per_sec":"1.445116864","p50_e2el":21.67807871,"p50_tpot":0.020152486,"p50_ttft":1.062086034,"p50_intvty":49.62167072},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":128,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":"32409554.555555556","tput_per_gpu":55420.33829,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"192.6989","input_tput_per_gpu":49082.91,"kv_cache_precision":"FP8","total_input_tokens":9458221,"output_tput_per_gpu":6110.477,"total_output_tokens":1177482,"tput_requests_per_sec":"6.642488","p50_e2el":20.67846,"p50_tpot":0.020028,"p50_ttft":0.190048,"p50_intvty":49.76625},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":10,"date":"2026-02-05","num_gpus":4,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":5115655.769590644,"tput_per_gpu":8747.771366,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"23.57523","input_tput_per_gpu":7775.448,"kv_cache_precision":"FP8","total_input_tokens":733232,"output_tput_per_gpu":972.3234,"total_output_tokens":91691,"tput_requests_per_sec":"4.24174","p50_e2el":2.466698,"p50_tpot":0.002347,"p50_ttft":0.065676,"p50_intvty":426.0686},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":8,"date":"2026-02-05","num_gpus":4,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":4409528.661403509,"tput_per_gpu":7540.294011,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"21.88237","input_tput_per_gpu":6694.076,"kv_cache_precision":"FP8","total_input_tokens":585929,"output_tput_per_gpu":846.2178,"total_output_tokens":74069,"tput_requests_per_sec":"3.655911","p50_e2el":2.313883,"p50_tpot":0.002199,"p50_ttft":0.064514,"p50_intvty":454.7942},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":4,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":952613.0754385965,"tput_per_gpu":1628.968359,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"12.80197","input_tput_per_gpu":1447.297,"kv_cache_precision":"FP8","total_input_tokens":74113,"output_tput_per_gpu":181.6713,"total_output_tokens":9303,"tput_requests_per_sec":"0.78113","p50_e2el":1.357177,"p50_tpot":0.001261,"p50_ttft":0.067069,"p50_intvty":792.9572},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":1,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":3,"tput_per_mw":"2767628.2128654975","tput_per_gpu":4732.644244,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"17.62566","input_tput_per_gpu":4204.834,"kv_cache_precision":"FP8","total_input_tokens":74113,"output_tput_per_gpu":527.8099,"total_output_tokens":9303,"tput_requests_per_sec":"0.567354","p50_e2el":1.631976,"p50_tpot":0.001501,"p50_ttft":0.096344,"p50_intvty":666.1751},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":8192,"osl":1024,"conc":2,"date":"2026-03-27","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":863670.0226277373,"dataset_name":"random","tput_per_gpu":1183.227931,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"19.47215697","input_tput_per_gpu":1051.758161,"total_input_tokens":81920,"output_tput_per_gpu":131.4697701,"total_output_tokens":10240,"tput_requests_per_sec":"0.51355379","p50_e2el":3.890964555,"p50_tpot":0.003665886,"p50_ttft":0.140763429,"p50_intvty":272.7853695},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":8,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"1476448.9583941607","dataset_name":"random","tput_per_gpu":2022.735073,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"22.78103574","input_tput_per_gpu":1797.986732,"total_input_tokens":327680,"output_tput_per_gpu":224.7483415,"total_output_tokens":40960,"tput_requests_per_sec":"1.755846418","p50_e2el":4.55006568,"p50_tpot":0.004138305,"p50_ttft":0.316579847,"p50_intvty":241.6448384},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":8192,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":2398605.535766423,"dataset_name":"random","tput_per_gpu":3286.089584,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"28.04549226","input_tput_per_gpu":2920.968519,"total_input_tokens":655360,"output_tput_per_gpu":365.1210649,"total_output_tokens":81920,"tput_requests_per_sec":"2.85250832","p50_e2el":5.594319984,"p50_tpot":0.004976116,"p50_ttft":0.50375361,"p50_intvty":200.9599571},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":384,"date":"2026-02-05","num_gpus":1,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":1,"tput_per_mw":38147542.61403509,"tput_per_gpu":65232.29787,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"488.5011","input_tput_per_gpu":57984.14,"kv_cache_precision":"FP8","total_input_tokens":28325314,"output_tput_per_gpu":7248.162,"total_output_tokens":3540735,"tput_requests_per_sec":"7.860781","p50_e2el":50.74934,"p50_tpot":0.039427,"p50_ttft":10.41593,"p50_intvty":25.36359},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":32,"date":"2026-02-05","num_gpus":2,"sequence":"8K/1K","framework":"TensorRT LLM","precision":"FP4","spec_length":2,"tput_per_mw":14845451.03508772,"tput_per_gpu":25385.72127,"model_display":"gpt-oss-120b","spec_decoding":"eagle","date_timestamp":1770278400,"serving_config":"agg","duration_seconds":"52.21762","input_tput_per_gpu":22550.49,"kv_cache_precision":"FP8","total_input_tokens":2355066,"output_tput_per_gpu":2835.231,"total_output_tokens":296098,"tput_requests_per_sec":"6.128199","p50_e2el":5.528305,"p50_tpot":0.005317,"p50_ttft":0.08912,"p50_intvty":188.0796},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"15817149.643752575","tput_per_gpu":27047.3258908169,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":14.8658844168},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"193282.19080923978","tput_per_gpu":330.5125462838,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":204.8590147099},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"3568184.4761803513","tput_per_gpu":6101.5954542684,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":79.9594965759},{"hw":"B200","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":8,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":41514.91393497076,"tput_per_gpu":70.9905028288,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":234.1828697817},{"ep":1,"hw":"B300","tp":8,"conc":1,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"1065963.1368421053","tput_per_gpu":2025.32996,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":2010.1993,"output_tput_per_gpu":15.13066,"p50_itl":0.00423,"p90_itl":0.00493,"mean_itl":0.00454,"p50_e2el":5.26484,"p50_tpot":0.00423,"p50_ttft":1.37168,"p90_e2el":23.00267,"p90_tpot":0.00493,"p90_ttft":3.13566,"mean_e2el":9.62951,"mean_tpot":0.00454,"mean_ttft":1.69433,"p50_intvty":"236.40661938534276","p90_intvty":"202.83975659229208","mean_intvty":"220.26431718061676"},{"ep":1,"hw":"B300","tp":8,"conc":32,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":5285715.010526315,"tput_per_gpu":10042.85852,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":9972.28716,"output_tput_per_gpu":70.57136,"p50_itl":0.03825,"p90_itl":0.05677,"mean_itl":0.04248,"p50_e2el":14.09638,"p50_tpot":0.03825,"p50_ttft":0.81717,"p90_e2el":86.65062,"p90_tpot":0.05677,"p90_ttft":2.17306,"mean_e2el":37.55724,"mean_tpot":0.04248,"mean_ttft":1.18799,"p50_intvty":"26.143790849673202","p90_intvty":17.61493746697199,"mean_intvty":23.54048964218456},{"ep":1,"hw":"B300","tp":8,"conc":2,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"1110511.4473684211","tput_per_gpu":2109.97175,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":2091.381,"output_tput_per_gpu":18.59075,"p50_itl":0.00435,"p90_itl":0.00755,"mean_itl":0.00551,"p50_e2el":3.085,"p50_tpot":0.00435,"p50_ttft":1.03472,"p90_e2el":22.3947,"p90_tpot":0.00755,"p90_ttft":2.40826,"mean_e2el":7.88723,"mean_tpot":0.00551,"mean_ttft":1.2983,"p50_intvty":229.8850574712644,"p90_intvty":132.4503311258278,"mean_intvty":"181.48820326678765"},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"212517.76575766082","tput_per_gpu":363.4053794456,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":152.6492332397},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":521927.8067543796,"tput_per_gpu":715.0410952535,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":65.7810178646},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":256,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"2004879.3551458397","tput_per_gpu":2746.6847165498,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":24.7752260273},{"hw":"B200","tp":4,"isl":8192,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":4,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"130492.42864070175","tput_per_gpu":223.1420529756,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":228.006119413},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":8,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"2260389.7788726315","tput_per_gpu":3865.2665218722,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":109.9829819651},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":41039.0700010219,"tput_per_gpu":56.2235259014,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":178.7958325421},{"ep":1,"hw":"GB300","tp":4,"conc":8,"date":"2026-09-10","num_gpus":4,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":5856311.655660377,"tput_per_gpu":12415.38071,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"serving_config":"agg","total_requests":1193,"duration_seconds":"3607.33787","input_tput_per_gpu":12322.00458,"total_input_tokens":178058503,"output_tput_per_gpu":93.37613,"total_output_tokens":1347358,"avg_output_tokens_expected":"1975.47378","theoretical_cache_hit_rate":"0.9771","stdev_output_tokens_expected":"2600.83957","avg_isl":"149034.81559","avg_osl":"1129.38558","p50_itl":0.00482,"p75_isl":"170969.0","p75_osl":"1132.0","p75_qps":"1.0","p90_isl":"338123.2","p90_itl":0.00685,"p90_osl":"2792.6","p90_qps":"1.0","p95_isl":"409253.6","p95_osl":"4450.4","p95_qps":"1.0","std_qps":"0.59625","mean_itl":0.00519,"mean_qps":"0.33056","p50_e2el":3.15098,"p50_tpot":0.00481,"p50_ttft":0.61036,"p90_e2el":13.93684,"p90_tpot":0.00685,"p90_ttft":1.0583,"mean_e2el":6.30946,"mean_tpot":0.00518,"mean_ttft":0.74874,"stdev_isl":"123990.34156","stdev_osl":"1918.75191","median_qps":"0.0","p50_intvty":207.4688796680498,"p90_intvty":"145.98540145985402","mean_intvty":"192.67822736030828"},{"ep":1,"hw":"B300","tp":8,"conc":8,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":"2726546.6263157893","tput_per_gpu":5180.43859,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":5139.80599,"output_tput_per_gpu":40.6326,"p50_itl":0.00782,"p90_itl":0.01565,"mean_itl":0.00947,"p50_e2el":5.41529,"p50_tpot":0.00782,"p50_ttft":1.04294,"p90_e2el":22.83942,"p90_tpot":0.01565,"p90_ttft":2.27809,"mean_e2el":10.22597,"mean_tpot":0.00947,"mean_ttft":1.31593,"p50_intvty":127.8772378516624,"p90_intvty":"63.897763578274756","mean_intvty":"105.59662090813094"},{"ep":1,"hw":"B300","tp":8,"conc":70,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":6613944.421052631,"tput_per_gpu":12566.4944,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":12478.92232,"output_tput_per_gpu":87.57208,"p50_itl":0.0912,"p90_itl":0.12677,"mean_itl":0.09484,"p50_e2el":37.66281,"p50_tpot":0.0912,"p50_ttft":1.21277,"p90_e2el":173.99393,"p90_tpot":0.12677,"p90_ttft":3.83755,"mean_e2el":80.73052,"mean_tpot":0.09484,"mean_ttft":2.25527,"p50_intvty":"10.964912280701753","p90_intvty":7.888301648655045,"mean_intvty":"10.544074230282583"},{"ep":1,"hw":"B300","tp":8,"conc":40,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":5503127.531578948,"tput_per_gpu":10455.94231,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":10380.17824,"output_tput_per_gpu":75.76407,"p50_itl":0.04746,"p90_itl":0.06571,"mean_itl":0.05043,"p50_e2el":16.72018,"p50_tpot":0.04746,"p50_ttft":0.8373,"p90_e2el":102.76093,"p90_tpot":0.06571,"p90_ttft":2.28419,"mean_e2el":44.06868,"mean_tpot":0.05043,"mean_ttft":1.2773,"p50_intvty":"21.070375052675935","p90_intvty":"15.218383807639627","mean_intvty":19.8294665873488},{"ep":1,"hw":"B300","tp":8,"conc":56,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":6280120.084210526,"tput_per_gpu":11932.22816,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":11850.62244,"output_tput_per_gpu":81.60571,"p50_itl":0.06747,"p90_itl":0.0904,"mean_itl":0.07161,"p50_e2el":"24.952900000000003","p50_tpot":0.06747,"p50_ttft":0.99782,"p90_e2el":140.53845,"p90_tpot":0.0904,"p90_ttft":2.82823,"mean_e2el":61.30808,"mean_tpot":0.07161,"mean_ttft":"1.6747099999999997","p50_intvty":"14.821402104639098","p90_intvty":"11.061946902654867","mean_intvty":"13.964530093562352"},{"ep":1,"hw":"B300","tp":8,"conc":24,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":4696110.289473684,"tput_per_gpu":8922.60955,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":8856.35555,"output_tput_per_gpu":66.25399,"p50_itl":0.0326,"p90_itl":0.04619,"mean_itl":0.03549,"p50_e2el":11.76783,"p50_tpot":0.0326,"p50_ttft":0.77014,"p90_e2el":77.01256,"p90_tpot":0.04619,"p90_ttft":1.92893,"mean_e2el":31.69264,"mean_tpot":0.03549,"mean_ttft":1.08775,"p50_intvty":"30.674846625766875","p90_intvty":"21.649707728945657","mean_intvty":28.17695125387433},{"ep":1,"hw":"B300","tp":8,"conc":48,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":6014736.315789473,"tput_per_gpu":11427.999,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":11352.2622,"output_tput_per_gpu":75.7368,"p50_itl":0.05935,"p90_itl":0.07984,"mean_itl":0.06298,"p50_e2el":20.91419,"p50_tpot":0.05935,"p50_ttft":0.90908,"p90_e2el":118.47028,"p90_tpot":0.07984,"p90_ttft":2.63183,"mean_e2el":52.02566,"mean_tpot":0.06298,"mean_ttft":1.50773,"p50_intvty":"16.849199663016005","p90_intvty":"12.525050100200401","mean_intvty":"15.878056525881233"},{"ep":1,"hw":"B300","tp":8,"conc":4,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":1642046.4,"tput_per_gpu":3119.88816,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":3096.44211,"output_tput_per_gpu":23.44605,"p50_itl":0.00504,"p90_itl":0.00826,"mean_itl":0.00625,"p50_e2el":3.15423,"p50_tpot":0.00504,"p50_ttft":0.95808,"p90_e2el":21.81182,"p90_tpot":0.00826,"p90_ttft":2.30494,"mean_e2el":8.05834,"mean_tpot":0.00625,"mean_ttft":1.24218,"p50_intvty":198.4126984126984,"p90_intvty":"121.06537530266344","mean_intvty":160.0},{"ep":1,"hw":"B300","tp":8,"conc":16,"date":"2026-08-16","num_gpus":8,"scenario":"agentic","framework":"vLLM","precision":"FP4","tput_per_mw":4026683.010526316,"tput_per_gpu":7650.69772,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"serving_config":"agg","input_tput_per_gpu":7592.28324,"output_tput_per_gpu":58.41448,"p50_itl":0.016,"p90_itl":0.03113,"mean_itl":0.01877,"p50_e2el":7.24245,"p50_tpot":0.016,"p50_ttft":1.11374,"p90_e2el":38.33878,"p90_tpot":0.03113,"p90_ttft":3.22504,"mean_e2el":17.70412,"mean_tpot":0.01877,"mean_ttft":1.67245,"p50_intvty":62.5,"p90_intvty":32.123353678124,"mean_intvty":53.27650506126798},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":6,"date":"2026-09-10","num_gpus":28,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":415789.1181818182,"tput_per_gpu":1372.10409,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789023600,"num_decode_gpu":24,"serving_config":"disagg","total_requests":873,"num_prefill_gpu":4,"duration_seconds":"3603.78117","input_tput_per_gpu":1361.70605,"total_input_tokens":137404137,"output_tput_per_gpu":10.39805,"total_output_tokens":1049224,"avg_output_tokens_expected":"1728.14412","theoretical_cache_hit_rate":"0.97726","stdev_output_tokens_expected":"2703.92891","avg_isl":"157393.05498","avg_osl":"1201.86025","p50_itl":0.00343,"p75_isl":"205575.0","p75_osl":"1276.0","p75_qps":"0.0","p90_isl":"388899.0","p90_itl":0.00362,"p90_osl":"2945.6","p90_qps":"1.0","p95_isl":"530309.6","p95_osl":"4435.6","p95_qps":"1.0","std_qps":"0.51731","mean_itl":0.00348,"mean_qps":"0.2457","p50_e2el":2.62188,"p50_tpot":0.00343,"p50_ttft":0.54795,"p90_e2el":11.3156,"p90_tpot":0.00363,"p90_ttft":1.31597,"mean_e2el":4.90702,"mean_tpot":0.00349,"mean_ttft":0.77165,"stdev_isl":"150914.10291","stdev_osl":"2026.3895","median_qps":"0.0","p50_intvty":291.5451895043732,"p90_intvty":"276.24309392265195","mean_intvty":"287.35632183908046"},{"ep":16,"hw":"GB300","tp":16,"conc":1440,"date":"2026-09-10","num_gpus":40,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":51504711.26415095,"tput_per_gpu":109189.98788,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":122430,"num_prefill_gpu":24,"duration_seconds":"3629.79697","input_tput_per_gpu":108364.87899,"total_input_tokens":3089686193,"output_tput_per_gpu":825.10889,"total_output_tokens":24109497,"avg_output_tokens_expected":"1343.56238","theoretical_cache_hit_rate":"0.96712","stdev_output_tokens_expected":"2566.13812","avg_isl":"128511.80584","avg_osl":"978.51107","p50_itl":0.0172,"p75_isl":"156598.75","p75_osl":"990.0","p75_qps":"42.0","p90_isl":"277834.4","p90_itl":0.02363,"p90_osl":"2258.0","p90_qps":"73.0","p95_isl":"414884.15","p95_osl":"3737.0","p95_qps":"108.9","std_qps":"32.08153","mean_itl":0.01771,"mean_qps":"33.79133","p50_e2el":22.07307,"p50_tpot":0.01718,"p50_ttft":10.41917,"p90_e2el":69.63615,"p90_tpot":0.02355,"p90_ttft":33.96058,"mean_e2el":33.08796,"mean_tpot":0.01764,"mean_ttft":16.30547,"stdev_isl":"132239.15123","stdev_osl":"1899.12817","median_qps":"30.0","p50_intvty":58.13953488372093,"p90_intvty":42.31908590774439,"mean_intvty":56.46527385657821},{"ep":16,"hw":"GB300","tp":16,"conc":1920,"date":"2026-09-10","num_gpus":48,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":53005751.31132075,"tput_per_gpu":112372.19278,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":153829,"num_prefill_gpu":32,"duration_seconds":"3629.96882","input_tput_per_gpu":111526.08901,"total_input_tokens":4984064794,"output_tput_per_gpu":846.10376,"total_output_tokens":31862058,"avg_output_tokens_expected":"1342.4461","theoretical_cache_hit_rate":"0.96682","stdev_output_tokens_expected":"2593.88077","avg_isl":"126322.98732","avg_osl":"958.3619","p50_itl":0.02319,"p75_isl":"154271.0","p75_osl":"971.0","p75_qps":"54.0","p90_isl":"272761.2","p90_itl":0.03156,"p90_osl":"2201.0","p90_qps":"99.0","p95_isl":"406457.6","p95_osl":"3664.0","p95_qps":"139.95","std_qps":"41.54971","mean_itl":0.02362,"mean_qps":"42.46604","p50_e2el":23.22554,"p50_tpot":0.02316,"p50_ttft":9.37965,"p90_e2el":78.24193,"p90_tpot":0.03146,"p90_ttft":30.95697,"mean_e2el":37.01443,"mean_tpot":0.02353,"mean_ttft":14.94226,"stdev_isl":"130230.92597","stdev_osl":"1847.30008","median_qps":"37.0","p50_intvty":43.122035360069,"p90_intvty":"31.685678073510775","mean_intvty":"42.337002540220155"},{"ep":16,"hw":"GB300","tp":16,"conc":480,"date":"2026-09-10","num_gpus":24,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"29708738.886792455","tput_per_gpu":62982.52644,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":41855,"num_prefill_gpu":8,"duration_seconds":"3629.70317","input_tput_per_gpu":62485.32608,"total_input_tokens":977240973,"output_tput_per_gpu":497.20037,"total_output_tokens":7829934,"avg_output_tokens_expected":"1376.08602","theoretical_cache_hit_rate":"0.96692","stdev_output_tokens_expected":"2622.46835","avg_isl":"130050.80557","avg_osl":"1034.82389","p50_itl":0.00875,"p75_isl":"160601.5","p75_osl":"1040.5","p75_qps":"15.0","p90_isl":"278315.2","p90_itl":0.01026,"p90_osl":"2379.6","p90_qps":"19.0","p95_isl":"418835.0","p95_osl":"3936.0","p95_qps":"22.0","std_qps":"6.20987","mean_itl":0.00899,"mean_qps":"11.55175","p50_e2el":24.62689,"p50_tpot":0.00874,"p50_ttft":17.70385,"p90_e2el":61.29706,"p90_tpot":0.01023,"p90_ttft":43.62573,"mean_e2el":32.30358,"mean_tpot":0.00896,"mean_ttft":23.28409,"stdev_isl":"135066.1189","stdev_osl":"2044.89105","median_qps":"11.0","p50_intvty":"114.28571428571428","p90_intvty":97.46588693957115,"mean_intvty":"111.23470522803115"},{"ep":16,"hw":"GB300","tp":16,"conc":960,"date":"2026-09-10","num_gpus":32,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo SGLang","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":43933668.16037736,"tput_per_gpu":93139.3765,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":82315,"num_prefill_gpu":16,"duration_seconds":"3629.91023","input_tput_per_gpu":92433.29901,"total_input_tokens":1947160954,"output_tput_per_gpu":706.07749,"total_output_tokens":16899336,"avg_output_tokens_expected":"1363.08424","theoretical_cache_hit_rate":"0.96771","stdev_output_tokens_expected":"2595.56091","avg_isl":"130435.35783","avg_osl":"996.3668","p50_itl":0.01267,"p75_isl":"158895.0","p75_osl":"995.0","p75_qps":"29.0","p90_isl":"283938.6","p90_itl":0.01702,"p90_osl":"2298.0","p90_qps":"41.0","p95_isl":"427377.0","p95_osl":"3821.0","p95_qps":"61.0","std_qps":"17.62472","mean_itl":0.0132,"mean_qps":"22.71985","p50_e2el":23.23239,"p50_tpot":0.01265,"p50_ttft":14.19937,"p90_e2el":67.6887,"p90_tpot":0.01692,"p90_ttft":40.6952,"mean_e2el":32.75053,"mean_tpot":0.01314,"mean_ttft":20.03124,"stdev_isl":"133493.02713","stdev_osl":"1974.16904","median_qps":"21.0","p50_intvty":78.92659826361484,"p90_intvty":58.75440658049354,"mean_intvty":75.75757575757576},{"ep":8,"hw":"B300","tp":8,"conc":32,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"11037119.042105263","tput_per_gpu":20970.52618,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":3833,"duration_seconds":"3620.17144","input_tput_per_gpu":20837.93446,"total_input_tokens":603495161,"output_tput_per_gpu":132.59172,"total_output_tokens":3840038,"avg_output_tokens_expected":"1418.43926","theoretical_cache_hit_rate":"0.97647","stdev_output_tokens_expected":"2161.75788","avg_isl":"157447.21132","avg_osl":"1001.83616","p50_itl":0.00821,"p75_isl":"203741.0","p75_osl":"1067.0","p75_qps":"2.0","p90_isl":"307169.8","p90_itl":"0.012230000000000001","p90_osl":"2505.8","p90_qps":"3.0","p95_isl":"390150.8","p95_osl":"3992.4","p95_qps":"3.0","std_qps":"1.19598","mean_itl":0.00877,"mean_qps":"1.05886","p50_e2el":7.03192,"p50_tpot":0.00822,"p50_ttft":2.03054,"p90_e2el":30.38555,"p90_tpot":"0.012230000000000001","p90_ttft":8.51293,"mean_e2el":13.05845,"mean_tpot":0.0088,"mean_ttft":3.8355,"stdev_isl":"139541.69935","stdev_osl":"1709.85944","median_qps":"1.0","p50_intvty":121.8026796589525,"p90_intvty":81.76614881439085,"mean_intvty":"114.02508551881414"},{"ep":1,"hw":"GB300","tp":8,"conc":4,"date":"2026-09-10","num_gpus":8,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"1842233.0990566039","tput_per_gpu":3905.53417,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"serving_config":"agg","total_requests":605,"duration_seconds":"3612.14488","input_tput_per_gpu":3878.13954,"total_input_tokens":112526158,"output_tput_per_gpu":27.39463,"total_output_tokens":791628,"avg_output_tokens_expected":"1883.13884","theoretical_cache_hit_rate":"0.97859","stdev_output_tokens_expected":"2634.22661","avg_isl":"185235.06612","avg_osl":"1308.47438","p50_itl":0.0034,"p75_isl":"252123.0","p75_osl":"1546.0","p75_qps":"0.0","p90_isl":"529410.2","p90_itl":0.00418,"p90_osl":"3962.0","p90_qps":"1.0","p95_isl":"592812.4","p95_osl":"5475.2","p95_qps":"1.0","std_qps":"0.40332","mean_itl":0.00367,"mean_qps":"0.16736","p50_e2el":2.16552,"p50_tpot":0.00339,"p50_ttft":0.52752,"p90_e2el":14.69614,"p90_tpot":0.00416,"p90_ttft":0.96646,"mean_e2el":5.25954,"mean_tpot":0.00365,"mean_ttft":0.69225,"stdev_isl":"182928.57324","stdev_osl":"2219.73583","median_qps":"0.0","p50_intvty":"294.11764705882354","p90_intvty":"239.23444976076556","mean_intvty":"272.47956403269757"},{"ep":8,"hw":"B300","tp":8,"conc":64,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"18031037.763157893","tput_per_gpu":34258.97175,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":7666,"duration_seconds":"3629.58696","input_tput_per_gpu":33995.94066,"total_input_tokens":987447878,"output_tput_per_gpu":263.0311,"total_output_tokens":7637555,"avg_output_tokens_expected":"1177.53024","theoretical_cache_hit_rate":"0.96928","stdev_output_tokens_expected":"2012.89686","avg_isl":"128767.25594","avg_osl":"996.28933","p50_itl":0.01267,"p75_isl":"151639.5","p75_osl":"1048.75","p75_qps":"3.0","p90_isl":"276524.0","p90_itl":0.01575,"p90_osl":"2263.0","p90_qps":"4.0","p95_isl":"503574.5","p95_osl":"3704.25","p95_qps":"5.0","std_qps":"1.76507","mean_itl":0.01251,"mean_qps":"2.11273","p50_e2el":11.70662,"p50_tpot":0.01268,"p50_ttft":3.80585,"p90_e2el":39.43505,"p90_tpot":0.01575,"p90_ttft":12.45745,"mean_e2el":19.08666,"mean_tpot":0.01255,"mean_ttft":5.93095,"stdev_isl":"148694.86764","stdev_osl":"1987.06055","median_qps":"2.0","p50_intvty":78.92659826361484,"p90_intvty":"63.492063492063494","mean_intvty":79.93605115907275},{"ep":1,"hw":"GB300","tp":8,"conc":1,"date":"2026-09-10","num_gpus":8,"scenario":"agentic","framework":"Dynamo SGLang","precision":"FP4","tput_per_mw":"1175859.4103773586","tput_per_gpu":2492.82195,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789023600,"serving_config":"agg","total_requests":266,"duration_seconds":"3594.35474","input_tput_per_gpu":2474.57531,"total_input_tokens":71488185,"output_tput_per_gpu":18.24663,"total_output_tokens":524680,"avg_output_tokens_expected":"2144.09398","theoretical_cache_hit_rate":"0.98067","stdev_output_tokens_expected":"2910.32424","avg_isl":"267503.80451","avg_osl":"1972.47744","p50_itl":0.0032,"p75_isl":"389712.0","p75_osl":"2737.0","p75_qps":"0.0","p90_isl":"477771.0","p90_itl":0.00347,"p90_osl":"5281.5","p90_qps":"0.0","p95_isl":"511615.75","p95_osl":"6908.25","p95_qps":"1.0","std_qps":"0.27202","mean_itl":0.0033,"mean_qps":"0.07388","p50_e2el":3.63665,"p50_tpot":0.0032,"p50_ttft":0.63294,"p90_e2el":17.7854,"p90_tpot":0.00347,"p90_ttft":1.74447,"mean_e2el":7.18468,"mean_tpot":0.0033,"mean_ttft":0.80755,"stdev_isl":"156247.01999","stdev_osl":"2651.64109","median_qps":"0.0","p50_intvty":312.5,"p90_intvty":288.1844380403458,"mean_intvty":"303.03030303030306"},{"ep":32,"hw":"Vera Rubin NVL72","tp":32,"conc":561,"date":"2026-09-09","num_gpus":48,"scenario":"agentic","decode_ep":32,"decode_tp":32,"framework":"TensorRT LLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":"20411917.863636363","tput_per_gpu":67359.32895,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788937200,"num_decode_gpu":32,"serving_config":"disagg","total_requests":81531,"num_prefill_gpu":16,"duration_seconds":"3629.34037","input_tput_per_gpu":66872.97579,"total_input_tokens":11649829951,"output_tput_per_gpu":486.35316,"total_output_tokens":84726776,"avg_output_tokens_expected":"1358.47461","theoretical_cache_hit_rate":"0.97132","stdev_output_tokens_expected":"2521.41561","avg_isl":"142888.34862","avg_osl":"1039.19707","p50_itl":0.00511,"p75_isl":"179916.0","p75_osl":"1025.0","p75_qps":"26.0","p90_isl":"308532.0","p90_itl":0.00532,"p90_osl":"2429.0","p90_qps":"30.0","p95_isl":"444858.0","p95_osl":"3984.0","p95_qps":"32.0","std_qps":"6.00402","mean_qps":"22.47863","p50_e2el":5.40627,"p50_tpot":0.00511,"p50_ttft":2.55257,"p90_e2el":17.05462,"p90_tpot":0.00532,"p90_ttft":5.53245,"stdev_isl":"141428.85625","stdev_osl":"2052.85252","median_qps":"23.0","p50_intvty":"195.69471624266146","p90_intvty":187.9699248120301},{"ep":16,"hw":"Vera Rubin NVL72","tp":16,"conc":2572,"date":"2026-09-09","num_gpus":48,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"TensorRT LLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":71484061.33636364,"tput_per_gpu":235897.40241,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788937200,"num_decode_gpu":16,"serving_config":"disagg","total_requests":299971,"num_prefill_gpu":32,"duration_seconds":"3629.87732","input_tput_per_gpu":234157.08444,"total_input_tokens":40798151582,"output_tput_per_gpu":1740.31797,"total_output_tokens":303222756,"avg_output_tokens_expected":"1336.72347","theoretical_cache_hit_rate":"0.96942","stdev_output_tokens_expected":"2526.21926","avg_isl":"136006.98595","avg_osl":"1010.84023","p50_itl":0.01141,"p75_isl":"169688.0","p75_osl":"1013.0","p75_qps":"92.0","p90_isl":"293045.0","p90_itl":0.01184,"p90_osl":"2357.0","p90_qps":"99.0","p95_isl":"427407.0","p95_osl":"3865.0","p95_qps":"103.0","std_qps":"15.54459","mean_qps":"82.70416","p50_e2el":9.89753,"p50_tpot":0.01141,"p50_ttft":3.88591,"p90_e2el":35.60785,"p90_tpot":0.01183,"p90_ttft":9.74736,"stdev_isl":"135707.61379","stdev_osl":"1967.08837","median_qps":"85.0","p50_intvty":87.64241893076249,"p90_intvty":84.45945945945947},{"ep":32,"hw":"Vera Rubin NVL72","tp":32,"conc":160,"date":"2026-09-09","num_gpus":40,"scenario":"agentic","decode_ep":32,"decode_tp":32,"framework":"TensorRT LLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":6790011.957575758,"tput_per_gpu":22407.03946,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788937200,"num_decode_gpu":32,"serving_config":"disagg","total_requests":22618,"num_prefill_gpu":8,"duration_seconds":"3623.57628","input_tput_per_gpu":22239.39156,"total_input_tokens":3223445271,"output_tput_per_gpu":167.6479,"total_output_tokens":24299398,"avg_output_tokens_expected":"1341.56492","theoretical_cache_hit_rate":"0.97146","stdev_output_tokens_expected":"2467.48719","avg_isl":"142516.81276","avg_osl":"1074.33893","p50_itl":0.00447,"p75_isl":"182054.25","p75_osl":"1039.0","p75_qps":"8.0","p90_isl":"298490.7","p90_itl":0.00464,"p90_osl":"2499.6","p90_qps":"10.0","p95_isl":"447171.1","p95_osl":"4119.0","p95_qps":"11.0","std_qps":"2.74079","mean_qps":"6.24262","p50_e2el":6.71063,"p50_tpot":0.00447,"p50_ttft":3.4994,"p90_e2el":17.32203,"p90_tpot":0.00464,"p90_ttft":8.93906,"stdev_isl":"140637.50117","stdev_osl":"2148.61817","median_qps":"6.0","p50_intvty":"223.71364653243847","p90_intvty":"215.51724137931035"},{"ep":4,"hw":"Vera Rubin NVL72","tp":4,"conc":24,"date":"2026-09-09","num_gpus":28,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":"1741111.1030303033","tput_per_gpu":5745.66664,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788937200,"num_decode_gpu":24,"serving_config":"disagg","total_requests":3660,"num_prefill_gpu":4,"duration_seconds":"3618.17199","input_tput_per_gpu":5709.16902,"total_input_tokens":578389152,"output_tput_per_gpu":36.49761,"total_output_tokens":3697530,"avg_output_tokens_expected":"1505.09826","theoretical_cache_hit_rate":"0.97698","stdev_output_tokens_expected":"2293.84837","avg_isl":"158029.82295","avg_osl":"1010.2541","p50_itl":0.00354,"p75_isl":"205203.5","p75_osl":"1031.0","p75_qps":"2.0","p90_isl":"313061.3","p90_itl":0.00419,"p90_osl":"2578.5","p90_qps":"2.0","p95_isl":"390626.85","p95_osl":"4154.45","p95_qps":"3.0","std_qps":"1.08372","mean_qps":"1.01245","p50_e2el":2.09992,"p50_tpot":0.00352,"p50_ttft":0.68864,"p90_e2el":9.51429,"p90_tpot":0.00419,"p90_ttft":1.39291,"stdev_isl":"135935.49351","stdev_osl":"1828.57344","median_qps":"1.0","p50_intvty":282.4858757062147,"p90_intvty":238.6634844868735},{"ep":8,"hw":"B300","tp":8,"conc":256,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":40494184.1,"tput_per_gpu":76938.94979,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":17539,"duration_seconds":"3629.31665","input_tput_per_gpu":76353.70053,"total_input_tokens":2217748412,"output_tput_per_gpu":585.24926,"total_output_tokens":16992441,"avg_output_tokens_expected":"1330.25951","theoretical_cache_hit_rate":"0.96627","stdev_output_tokens_expected":"2454.64491","avg_isl":"126397.9732","avg_osl":"968.83739","p50_itl":0.02054,"p75_isl":"152133.5","p75_osl":"958.0","p75_qps":"7.0","p90_isl":"270302.2","p90_itl":0.02362,"p90_osl":"2216.2","p90_qps":"9.0","p95_isl":"403664.4","p95_osl":"3705.0","p95_qps":"11.0","std_qps":"3.46307","mean_itl":0.01975,"mean_qps":"4.83407","p50_e2el":29.72998,"p50_tpot":0.02055,"p50_ttft":13.37225,"p90_e2el":103.27687,"p90_tpot":0.02363,"p90_ttft":62.51198,"mean_e2el":46.74552,"mean_tpot":0.01983,"mean_ttft":26.3922,"stdev_isl":"128333.94248","stdev_osl":"1973.28122","median_qps":"4.0","p50_intvty":"48.685491723466406","p90_intvty":"42.337002540220155","mean_intvty":50.63291139240506},{"ep":1,"hw":"B300","tp":8,"conc":8,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":3527479.652631579,"tput_per_gpu":6702.21134,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":1291,"duration_seconds":"3610.08261","input_tput_per_gpu":6652.49108,"total_input_tokens":192128339,"output_tput_per_gpu":49.72026,"total_output_tokens":1435954,"avg_output_tokens_expected":"1961.66582","theoretical_cache_hit_rate":"0.97698","stdev_output_tokens_expected":"2560.42436","avg_isl":"148821.33153","avg_osl":"1112.2804","p50_itl":0.00431,"p75_isl":"168177.0","p75_osl":"1128.5","p75_qps":"1.0","p90_isl":"344665.0","p90_itl":0.00626,"p90_osl":"2795.0","p90_qps":"1.0","p95_isl":"441367.5","p95_osl":"4470.5","p95_qps":"2.0","std_qps":"0.62437","mean_itl":0.00472,"mean_qps":"0.35744","p50_e2el":2.72598,"p50_tpot":0.00431,"p50_ttft":0.51314,"p90_e2el":13.58825,"p90_tpot":0.00626,"p90_ttft":1.38459,"mean_e2el":5.62625,"mean_tpot":0.00472,"mean_ttft":0.72779,"stdev_isl":"126936.66422","stdev_osl":"1884.25802","median_qps":"0.0","p50_intvty":"232.01856148491882","p90_intvty":159.7444089456869,"mean_intvty":211.864406779661},{"ep":8,"hw":"B300","tp":8,"conc":128,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"26053131.126315787","tput_per_gpu":49500.94914,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":12242,"duration_seconds":"3629.9156","input_tput_per_gpu":49072.36734,"total_input_tokens":1425028415,"output_tput_per_gpu":428.5818,"total_output_tokens":12445726,"avg_output_tokens_expected":"1241.32291","theoretical_cache_hit_rate":"0.96374","stdev_output_tokens_expected":"2340.23388","avg_isl":"116404.86971","avg_osl":"1016.64156","p50_itl":0.01606,"p75_isl":"141951.75","p75_osl":"1071.0","p75_qps":"5.0","p90_isl":"247811.1","p90_itl":0.01855,"p90_osl":"2260.0","p90_qps":"7.0","p95_isl":"371584.15","p95_osl":"3735.7","p95_qps":"8.0","std_qps":"2.50762","mean_itl":0.01576,"mean_qps":"3.37404","p50_e2el":18.55529,"p50_tpot":0.01607,"p50_ttft":6.88383,"p90_e2el":59.47092,"p90_tpot":0.01855,"p90_ttft":27.33605,"mean_e2el":29.24148,"mean_tpot":0.01581,"mean_ttft":12.30845,"stdev_isl":"129215.57197","stdev_osl":"1964.86418","median_qps":"3.0","p50_intvty":62.266500622665,"p90_intvty":"53.908355795148246","mean_intvty":63.45177664974619},{"ep":1,"hw":"B300","tp":8,"conc":1,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":1264222.152631579,"tput_per_gpu":2402.02209,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":261,"duration_seconds":"3590.52761","input_tput_per_gpu":2384.35533,"total_input_tokens":68820920,"output_tput_per_gpu":17.66676,"total_output_tokens":507465,"avg_output_tokens_expected":"2071.99617","theoretical_cache_hit_rate":"0.98006","stdev_output_tokens_expected":"2783.64191","avg_isl":"262409.0","avg_osl":"1944.30651","p50_itl":0.00329,"p75_isl":"384291.0","p75_osl":"2633.0","p75_qps":"0.0","p90_isl":"466626.0","p90_itl":0.00354,"p90_osl":"5071.0","p90_qps":"0.0","p95_isl":"498038.0","p95_osl":"6910.0","p95_qps":"1.0","std_qps":"0.2658","mean_itl":0.00335,"mean_qps":"0.07256","p50_e2el":3.77051,"p50_tpot":0.00329,"p50_ttft":0.90398,"p90_e2el":17.6508,"p90_tpot":0.00354,"p90_ttft":1.81306,"mean_e2el":7.47535,"mean_tpot":0.00335,"mean_ttft":1.02718,"stdev_isl":"153293.99755","stdev_osl":"2646.21126","median_qps":"0.0","p50_intvty":303.951367781155,"p90_intvty":282.4858757062147,"mean_intvty":298.5074626865672},{"ep":1,"hw":"B300","tp":8,"conc":16,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":6383108.721052631,"tput_per_gpu":12127.90657,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":2406,"duration_seconds":"3602.73831","input_tput_per_gpu":12046.2116,"total_input_tokens":347239579,"output_tput_per_gpu":81.69498,"total_output_tokens":2354606,"avg_output_tokens_expected":"1593.34352","theoretical_cache_hit_rate":"0.97434","stdev_output_tokens_expected":"2476.18481","avg_isl":"144303.734","avg_osl":"978.63882","p50_itl":0.00593,"p75_isl":"169362.75","p75_osl":"892.5","p75_qps":"1.0","p90_isl":"257707.5","p90_itl":0.00954,"p90_osl":"2421.0","p90_qps":"2.0","p95_isl":"495370.75","p95_osl":"4271.75","p95_qps":"3.0","std_qps":"0.91766","mean_itl":0.00715,"mean_qps":"0.66768","p50_e2el":3.16274,"p50_tpot":0.00593,"p50_ttft":0.56854,"p90_e2el":16.06899,"p90_tpot":0.00954,"p90_ttft":1.75169,"mean_e2el":7.12635,"mean_tpot":0.00715,"mean_ttft":0.90079,"stdev_isl":"144793.1191","stdev_osl":"1975.72073","median_qps":"0.0","p50_intvty":"168.63406408094434","p90_intvty":"104.82180293501048","mean_intvty":"139.86013986013987"},{"ep":1,"hw":"B300","tp":8,"conc":4,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":1966149.752631579,"tput_per_gpu":3735.68453,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":590,"duration_seconds":"3598.54078","input_tput_per_gpu":3709.67073,"total_input_tokens":106795211,"output_tput_per_gpu":26.01381,"total_output_tokens":748894,"avg_output_tokens_expected":"1882.62373","theoretical_cache_hit_rate":"0.97813","stdev_output_tokens_expected":"2655.18046","avg_isl":"181008.8322","avg_osl":"1269.31186","p50_itl":0.00342,"p75_isl":"246544.5","p75_osl":"1402.0","p75_qps":"0.0","p90_isl":"521701.4","p90_itl":0.00433,"p90_osl":"3767.1","p90_qps":"1.0","p95_isl":"579693.1","p95_osl":"5355.85","p95_qps":"1.0","std_qps":"0.40003","mean_itl":0.00384,"mean_qps":"0.16411","p50_e2el":2.30222,"p50_tpot":0.00342,"p50_ttft":0.52337,"p90_e2el":13.90548,"p90_tpot":0.00433,"p90_ttft":1.52817,"mean_e2el":5.2937,"mean_tpot":0.00384,"mean_ttft":0.77161,"stdev_isl":"179880.38237","stdev_osl":"2195.80503","median_qps":"0.0","p50_intvty":"292.39766081871346","p90_intvty":230.9468822170901,"mean_intvty":260.4166666666667},{"ep":16,"hw":"Vera Rubin NVL72","tp":16,"conc":934,"date":"2026-09-09","num_gpus":32,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"TensorRT LLM","precision":"FP4","prefill_ep":8,"prefill_tp":8,"tput_per_mw":45976148.09090909,"tput_per_gpu":151721.2887,"model_display":"DeepSeek-V4-Pro","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788937200,"num_decode_gpu":16,"serving_config":"disagg","total_requests":126042,"num_prefill_gpu":16,"duration_seconds":"3629.34198","input_tput_per_gpu":150612.21607,"total_input_tokens":17491943646,"output_tput_per_gpu":1109.07263,"total_output_tokens":128806523,"avg_output_tokens_expected":"1343.53685","theoretical_cache_hit_rate":"0.97041","stdev_output_tokens_expected":"2510.29789","avg_isl":"138778.69001","avg_osl":"1021.93335","p50_itl":0.00677,"p75_isl":"174877.5","p75_osl":"1018.0","p75_qps":"40.0","p90_isl":"299668.8","p90_itl":0.00706,"p90_osl":"2377.0","p90_qps":"44.0","p95_isl":"434816.5","p95_osl":"3916.0","p95_qps":"47.0","std_qps":"8.08819","mean_qps":"34.76034","p50_e2el":7.31979,"p50_tpot":0.00677,"p50_ttft":3.63982,"p90_e2el":23.1179,"p90_tpot":0.00706,"p90_ttft":7.53959,"stdev_isl":"137037.84207","stdev_osl":"1991.13716","median_qps":"35.0","p50_intvty":"147.71048744460856","p90_intvty":"141.64305949008497"},{"ep":1,"hw":"GB300","tp":8,"conc":4,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1605561.0094339622","tput_per_gpu":3403.78934,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":563,"duration_seconds":"3601.74945","input_tput_per_gpu":3378.49146,"total_input_tokens":98066592,"output_tput_per_gpu":25.29788,"total_output_tokens":728465,"avg_output_tokens_expected":"1872.99822","theoretical_cache_hit_rate":"0.97618","stdev_output_tokens_expected":"2679.51844","avg_isl":"172909.12611","avg_osl":"1294.73002","p50_itl":0.00412,"p75_isl":"214317.0","p75_osl":"1496.5","p75_qps":"0.0","p90_isl":"515099.4","p90_itl":0.00495,"p90_osl":"3888.4","p90_qps":"1.0","p95_isl":"569039.7","p95_osl":"5411.2","p95_qps":"1.0","std_qps":"0.38999","mean_itl":0.00438,"mean_qps":"0.15655","p50_e2el":2.74296,"p50_tpot":0.00413,"p50_ttft":0.57972,"p90_e2el":19.9251,"p90_tpot":0.004939999999999999,"p90_ttft":2.22093,"mean_e2el":7.13567,"mean_tpot":0.00441,"mean_ttft":1.44147,"stdev_isl":"175938.2309","stdev_osl":"2238.25318","median_qps":"0.0","p50_intvty":"242.71844660194174","p90_intvty":202.020202020202,"mean_intvty":"228.31050228310502"},{"ep":1,"hw":"B300","tp":4,"conc":20,"date":"2026-08-19","num_gpus":4,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"18342004.994736843","tput_per_gpu":34849.80949,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":3078,"duration_seconds":"3602.21584","input_tput_per_gpu":34626.63054,"total_input_tokens":498930388,"output_tput_per_gpu":223.17895,"total_output_tokens":3215755,"avg_output_tokens_expected":"1596.40874","theoretical_cache_hit_rate":"0.97795","stdev_output_tokens_expected":"2338.92696","avg_isl":"162095.64263","avg_osl":"1044.75471","p50_itl":0.00133,"p75_isl":"205395.25","p75_osl":"1096.25","p75_qps":"1.0","p90_isl":"314927.2","p90_itl":0.00431,"p90_osl":"2666.3","p90_qps":"2.0","p95_isl":"400276.65","p95_osl":"4281.7","p95_qps":"3.0","std_qps":"1.01727","mean_qps":"0.85663","p50_e2el":2.40358,"p50_tpot":0.00297,"p50_ttft":1.49854,"p90_e2el":13.27152,"p90_tpot":0.00476,"p90_ttft":2.50065,"stdev_isl":"146761.28765","stdev_osl":"1812.86649","median_qps":"1.0","p50_intvty":751.8796992481203,"p90_intvty":"232.01856148491882"},{"ep":1,"hw":"B300","tp":4,"conc":30,"date":"2026-08-19","num_gpus":4,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"24045020.994736843","tput_per_gpu":45685.53989,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":4159,"duration_seconds":"3624.22466","input_tput_per_gpu":45394.48757,"total_input_tokens":658079285,"output_tput_per_gpu":291.05232,"total_output_tokens":4219356,"avg_output_tokens_expected":"1433.66005","theoretical_cache_hit_rate":"0.97727","stdev_output_tokens_expected":"2130.99757","avg_isl":"158230.17192","avg_osl":"1014.51214","p50_itl":0.00187,"p75_isl":"217663.0","p75_osl":"1090.5","p75_qps":"2.0","p90_isl":"317965.2","p90_itl":0.00614,"p90_osl":"2582.0","p90_qps":"3.0","p95_isl":"381986.0","p95_osl":"3999.1","p95_qps":"3.0","std_qps":"1.22163","mean_qps":"1.14925","p50_e2el":3.2341,"p50_tpot":0.00401,"p50_ttft":1.93471,"p90_e2el":18.51018,"p90_tpot":0.00725,"p90_ttft":3.44667,"stdev_isl":"132457.54213","stdev_osl":"1696.84195","median_qps":"1.0","p50_intvty":534.75935828877,"p90_intvty":"162.86644951140067"},{"ep":1,"hw":"B300","tp":2,"conc":20,"date":"2026-08-19","num_gpus":2,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"31773626.836842105","tput_per_gpu":60369.89099,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":2866,"duration_seconds":"3614.23609","input_tput_per_gpu":59954.25129,"total_input_tokens":433377638,"output_tput_per_gpu":415.6397,"total_output_tokens":3004440,"avg_output_tokens_expected":"1616.71885","theoretical_cache_hit_rate":"0.97561","stdev_output_tokens_expected":"2349.52647","avg_isl":"151213.41172","avg_osl":"1048.30426","p50_itl":0.00195,"p75_isl":"190474.25","p75_osl":"1069.0","p75_qps":"1.0","p90_isl":"289421.0","p90_itl":0.00677,"p90_osl":"2745.5","p90_qps":"2.0","p95_isl":"363082.0","p95_osl":"4383.0","p95_qps":"3.0","std_qps":"0.98266","mean_qps":"0.79407","p50_e2el":3.45962,"p50_tpot":0.00457,"p50_ttft":2.1501,"p90_e2el":20.28933,"p90_tpot":0.00753,"p90_ttft":3.51262,"stdev_isl":"135804.71575","stdev_osl":"1843.34753","median_qps":"1.0","p50_intvty":512.8205128205128,"p90_intvty":"147.71048744460856"},{"ep":1,"hw":"B300","tp":4,"conc":15,"date":"2026-08-19","num_gpus":4,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"11815490.563157894","tput_per_gpu":22449.43207,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":2508,"duration_seconds":"3609.74809","input_tput_per_gpu":22277.28799,"total_input_tokens":321661591,"output_tput_per_gpu":172.14408,"total_output_tokens":2485587,"avg_output_tokens_expected":"1659.72461","theoretical_cache_hit_rate":"0.97073","stdev_output_tokens_expected":"2507.8499","avg_isl":"128254.22289","avg_osl":"991.0634","p50_itl":0.00076,"p75_isl":"164685.75","p75_osl":"906.0","p75_qps":"1.0","p90_isl":"257748.5","p90_itl":0.0035,"p90_osl":"2521.9","p90_qps":"2.0","p95_isl":"323597.9","p95_osl":"4369.8","p95_qps":"2.0","std_qps":"0.89633","mean_qps":"0.69465","p50_e2el":1.84373,"p50_tpot":0.00245,"p50_ttft":1.27147,"p90_e2el":10.49962,"p90_tpot":0.00392,"p90_ttft":1.97243,"stdev_isl":"100356.90945","stdev_osl":"1983.48935","median_qps":"0.0","p50_intvty":"1315.7894736842104","p90_intvty":285.7142857142857},{"ep":1,"hw":"B300","tp":4,"conc":5,"date":"2026-08-19","num_gpus":4,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":4740635.457894737,"tput_per_gpu":9007.20737,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":833,"duration_seconds":"3600.02731","input_tput_per_gpu":8936.54228,"total_input_tokens":128687185,"output_tput_per_gpu":70.66509,"total_output_tokens":1017585,"avg_output_tokens_expected":"1681.2497","theoretical_cache_hit_rate":"0.97481","stdev_output_tokens_expected":"2636.7771","avg_isl":"154486.41657","avg_osl":"1221.59064","p50_itl":0.00147,"p75_isl":"189097.0","p75_osl":"1255.0","p75_qps":"0.0","p90_isl":"389768.2","p90_itl":0.00271,"p90_osl":"3067.6","p90_qps":"1.0","p95_isl":"548486.0","p95_osl":"4786.0","p95_qps":"1.0","std_qps":"0.49836","mean_qps":"0.23124","p50_e2el":2.03726,"p50_tpot":0.00198,"p50_ttft":1.06464,"p90_e2el":10.08489,"p90_tpot":0.0028,"p90_ttft":1.91232,"stdev_isl":"155780.31321","stdev_osl":"2039.01053","median_qps":"0.0","p50_intvty":680.2721088435375,"p90_intvty":"369.00369003690037"},{"ep":1,"hw":"B300","tp":4,"conc":10,"date":"2026-08-19","num_gpus":4,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":9019006.5,"tput_per_gpu":17136.11235,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":1547,"duration_seconds":"3603.02982","input_tput_per_gpu":17009.73024,"total_input_tokens":245146261,"output_tput_per_gpu":126.38211,"total_output_tokens":1821434,"avg_output_tokens_expected":"2027.14001","theoretical_cache_hit_rate":"0.97747","stdev_output_tokens_expected":"2783.15171","avg_isl":"158465.58565","avg_osl":"1177.39754","p50_itl":0.00121,"p75_isl":"189419.5","p75_osl":"1195.5","p75_qps":"1.0","p90_isl":"377968.8","p90_itl":0.00313,"p90_osl":"2948.4","p90_qps":"1.0","p95_isl":"489039.7","p95_osl":"4788.2","p95_qps":"2.0","std_qps":"0.68413","mean_qps":"0.4298","p50_e2el":1.93744,"p50_tpot":0.00232,"p50_ttft":1.19508,"p90_e2el":10.82749,"p90_tpot":0.00331,"p90_ttft":1.90892,"stdev_isl":"134348.92745","stdev_osl":"2056.76349","median_qps":"0.0","p50_intvty":826.4462809917355,"p90_intvty":319.4888178913738},{"ep":1,"hw":"B300","tp":4,"conc":25,"date":"2026-08-19","num_gpus":4,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":22603757.70526316,"tput_per_gpu":42947.13964,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":3881,"duration_seconds":"3613.39516","input_tput_per_gpu":42680.11522,"total_input_tokens":616880487,"output_tput_per_gpu":267.02442,"total_output_tokens":3859459,"avg_output_tokens_expected":"1464.58263","theoretical_cache_hit_rate":"0.9775","stdev_output_tokens_expected":"2227.82523","avg_isl":"158948.85004","avg_osl":"994.44963","p50_itl":0.00135,"p75_isl":"209907.0","p75_osl":"1030.0","p75_qps":"2.0","p90_isl":"318893.0","p90_itl":0.00495,"p90_osl":"2545.0","p90_qps":"3.0","p95_isl":"394890.0","p95_osl":"4045.0","p95_qps":"3.0","std_qps":"1.15103","mean_qps":"1.07658","p50_e2el":2.639,"p50_tpot":0.00341,"p50_ttft":1.69357,"p90_e2el":14.63111,"p90_tpot":0.00548,"p90_ttft":"2.7715799999999997","stdev_isl":"139491.56959","stdev_osl":"1738.33772","median_qps":"1.0","p50_intvty":740.7407407407406,"p90_intvty":202.020202020202},{"ep":1,"hw":"GB300","tp":8,"conc":56,"date":"2026-09-15","num_gpus":16,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":7085519.127358491,"tput_per_gpu":15021.30055,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":6493,"num_prefill_gpu":8,"duration_seconds":"3627.5258","input_tput_per_gpu":14908.82095,"total_input_tokens":995759252,"output_tput_per_gpu":112.47959,"total_output_tokens":6583685,"avg_output_tokens_expected":"1164.64767","theoretical_cache_hit_rate":"0.97052","stdev_output_tokens_expected":"1868.24714","avg_isl":"133268.76975","avg_osl":"1005.44617","p50_itl":0.01642,"p75_isl":"161997.0","p75_osl":"1059.0","p75_qps":"3.0","p90_isl":"287191.6","p90_itl":0.01799,"p90_osl":"2326.2","p90_qps":"4.0","p95_isl":"509312.8","p95_osl":"3755.6","p95_qps":"4.0","std_qps":"1.35899","mean_itl":0.0159,"mean_qps":"1.78963","p50_e2el":9.71692,"p50_tpot":0.01641,"p50_ttft":1.50612,"p90_e2el":40.21942,"p90_tpot":0.01798,"p90_ttft":4.63461,"mean_e2el":18.52394,"mean_tpot":0.0159,"mean_ttft":2.40144,"stdev_isl":"147037.80107","stdev_osl":"2015.90202","median_qps":"2.0","p50_intvty":60.90133982947625,"p90_intvty":55.58643690939411,"mean_intvty":"62.893081761006286"},{"ep":1,"hw":"GB300","tp":8,"conc":64,"date":"2026-09-15","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":6076488.419811321,"tput_per_gpu":12882.15545,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":8489,"num_prefill_gpu":8,"duration_seconds":"3629.75119","input_tput_per_gpu":12785.34981,"total_input_tokens":1115302458,"output_tput_per_gpu":96.80564,"total_output_tokens":8450762,"avg_output_tokens_expected":"1202.67628","theoretical_cache_hit_rate":"0.96959","stdev_output_tokens_expected":"2128.29364","avg_isl":"131203.12499","avg_osl":"993.41842","p50_itl":0.01251,"p75_isl":"155904.0","p75_osl":"1052.0","p75_qps":"3.0","p90_isl":"284193.0","p90_itl":0.01439,"p90_osl":"2244.0","p90_qps":"5.0","p95_isl":"510294.4","p95_osl":"3670.6","p95_qps":"5.0","std_qps":"1.64458","mean_itl":0.01202,"mean_qps":"2.33894","p50_e2el":9.54796,"p50_tpot":0.0125,"p50_ttft":2.3923,"p90_e2el":32.2154,"p90_tpot":0.01437,"p90_ttft":8.40514,"mean_e2el":15.89781,"mean_tpot":0.01203,"mean_ttft":3.93438,"stdev_isl":"151894.50272","stdev_osl":"1941.72252","median_qps":"2.0","p50_intvty":79.93605115907275,"p90_intvty":69.49270326615705,"mean_intvty":83.19467554076539},{"ep":1,"hw":"GB300","tp":8,"conc":48,"date":"2026-09-15","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":5063406.094339623,"tput_per_gpu":10734.42092,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":6050,"num_prefill_gpu":8,"duration_seconds":"3628.60182","input_tput_per_gpu":10663.34992,"total_input_tokens":929041936,"output_tput_per_gpu":71.071,"total_output_tokens":6194945,"avg_output_tokens_expected":"1237.83065","theoretical_cache_hit_rate":"0.97513","stdev_output_tokens_expected":"1958.12957","avg_isl":"153493.09455","avg_osl":"1023.02826","p50_itl":0.00926,"p75_isl":"200752.5","p75_osl":"1045.0","p75_qps":"2.0","p90_isl":"318526.9","p90_itl":0.0117,"p90_osl":"2423.1","p90_qps":"3.0","p95_isl":"561368.15","p95_osl":"3900.55","p95_qps":"4.0","std_qps":"1.3566","mean_itl":0.00907,"mean_qps":"1.66749","p50_e2el":6.2177,"p50_tpot":0.00925,"p50_ttft":1.41992,"p90_e2el":24.24845,"p90_tpot":0.01169,"p90_ttft":4.61475,"mean_e2el":11.50894,"mean_tpot":0.00906,"mean_ttft":2.18769,"stdev_isl":"158532.02253","stdev_osl":"2121.98695","median_qps":"1.0","p50_intvty":"107.99136069114472","p90_intvty":85.47008547008546,"mean_intvty":"110.25358324145535"},{"ep":1,"hw":"GB300","tp":8,"conc":48,"date":"2026-09-15","num_gpus":8,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":9105118.981132075,"tput_per_gpu":19302.85224,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"serving_config":"agg","total_requests":4077,"duration_seconds":"3626.8066","input_tput_per_gpu":19167.22907,"total_input_tokens":558273120,"output_tput_per_gpu":135.62317,"total_output_tokens":3955030,"avg_output_tokens_expected":"1272.79378","theoretical_cache_hit_rate":"0.97208","stdev_output_tokens_expected":"1984.73706","avg_isl":"136405.85308","avg_osl":"965.17832","p50_itl":0.02619,"p75_isl":"170355.0","p75_osl":"977.0","p75_qps":"2.0","p90_isl":"271775.0","p90_itl":0.03864,"p90_osl":"2199.2","p90_qps":"3.0","p95_isl":"498943.2","p95_osl":"3655.8","p95_qps":"4.0","std_qps":"1.23469","mean_itl":0.02727,"mean_qps":"1.12414","p50_e2el":11.93834,"p50_tpot":0.02623,"p50_ttft":1.15309,"p90_e2el":64.39693,"p90_tpot":0.03863,"p90_ttft":2.8768,"mean_e2el":28.49172,"mean_tpot":"0.027420000000000003","mean_ttft":1.71074,"stdev_isl":"136920.99515","stdev_osl":"2230.00028","median_qps":"1.0","p50_intvty":38.18251240931653,"p90_intvty":25.87991718426501,"mean_intvty":36.67033370003667},{"ep":1,"hw":"B300","tp":2,"conc":30,"date":"2026-08-19","num_gpus":2,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":40658980.62631579,"tput_per_gpu":77252.06319,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":3750,"duration_seconds":"3622.14868","input_tput_per_gpu":76746.16072,"total_input_tokens":555972010,"output_tput_per_gpu":505.90248,"total_output_tokens":3664908,"avg_output_tokens_expected":"1425.64751","theoretical_cache_hit_rate":"0.97536","stdev_output_tokens_expected":"2148.96781","avg_isl":"148259.20267","avg_osl":"977.3088","p50_itl":0.00259,"p75_isl":"201585.5","p75_osl":"1038.0","p75_qps":"2.0","p90_isl":"293072.4","p90_itl":0.01041,"p90_osl":"2479.4","p90_qps":"2.0","p95_isl":"356064.45","p95_osl":"3895.65","p95_qps":"3.0","std_qps":"1.07697","mean_qps":"1.03678","p50_e2el":5.65514,"p50_tpot":0.00657,"p50_ttft":3.06135,"p90_e2el":28.73819,"p90_tpot":0.012070000000000001,"p90_ttft":7.49451,"stdev_isl":"120743.32267","stdev_osl":"1675.15187","median_qps":"1.0","p50_intvty":386.1003861003861,"p90_intvty":96.06147934678194},{"ep":1,"hw":"B300","tp":2,"conc":25,"date":"2026-08-19","num_gpus":2,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":38594282.32105263,"tput_per_gpu":73329.13641,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":3528,"duration_seconds":"3611.53488","input_tput_per_gpu":72846.66204,"total_input_tokens":526176522,"output_tput_per_gpu":482.47437,"total_output_tokens":3484946,"avg_output_tokens_expected":"1454.8445","theoretical_cache_hit_rate":"0.97647","stdev_output_tokens_expected":"2189.43676","avg_isl":"149143.0051","avg_osl":"987.79649","p50_itl":0.0018,"p75_isl":"196896.0","p75_osl":"1010.5","p75_qps":"2.0","p90_isl":"295961.5","p90_itl":0.00812,"p90_osl":"2524.8","p90_qps":"2.0","p95_isl":"365569.75","p95_osl":"4138.3","p95_qps":"3.0","std_qps":"1.10252","mean_qps":"0.97809","p50_e2el":3.87564,"p50_tpot":0.00548,"p50_ttft":2.49844,"p90_e2el":23.42091,"p90_tpot":0.00898,"p90_ttft":4.15713,"stdev_isl":"127032.29127","stdev_osl":"1756.35539","median_qps":"1.0","p50_intvty":555.5555555555555,"p90_intvty":"123.15270935960591"},{"ep":1,"hw":"B300","tp":8,"conc":5,"date":"2026-08-19","num_gpus":8,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"2468695.8789473684","tput_per_gpu":4690.52217,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":863,"duration_seconds":"3600.14666","input_tput_per_gpu":4653.77697,"total_input_tokens":134034237,"output_tput_per_gpu":36.7452,"total_output_tokens":1058305,"avg_output_tokens_expected":"1665.54809","theoretical_cache_hit_rate":"0.97475","stdev_output_tokens_expected":"2602.59539","avg_isl":"155311.97798","avg_osl":"1226.30939","p50_itl":0.00134,"p75_isl":"189708.5","p75_osl":"1283.5","p75_qps":"0.0","p90_isl":"393679.2","p90_itl":0.00243,"p90_osl":"3139.2","p90_qps":"1.0","p95_isl":"556806.8","p95_osl":"4752.6","p95_qps":"1.0","std_qps":"0.51257","mean_qps":"0.23951","p50_e2el":1.86628,"p50_tpot":0.00185,"p50_ttft":0.9666,"p90_e2el":9.6488,"p90_tpot":0.00254,"p90_ttft":1.78778,"stdev_isl":"159503.11583","stdev_osl":"2022.42764","median_qps":"0.0","p50_intvty":746.2686567164179,"p90_intvty":"411.52263374485597"},{"ep":1,"hw":"B300","tp":2,"conc":30,"date":"2026-08-21","num_gpus":2,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":40834927.84210526,"tput_per_gpu":77586.3629,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787295600,"serving_config":"agg","total_requests":3761,"duration_seconds":"3623.82665","input_tput_per_gpu":77078.95066,"total_input_tokens":558641511,"output_tput_per_gpu":507.41224,"total_output_tokens":3677548,"avg_output_tokens_expected":"1427.84655","theoretical_cache_hit_rate":"0.97551","stdev_output_tokens_expected":"2153.70217","avg_isl":"148535.36586","avg_osl":"977.81122","p50_itl":0.00805,"p75_isl":"201828.0","p75_osl":"1038.0","p75_qps":"2.0","p90_isl":"293555.0","p90_itl":0.01475,"p90_osl":"2479.0","p90_qps":"2.0","p95_isl":"357245.0","p95_osl":"3897.0","p95_qps":"3.0","std_qps":"1.07593","mean_qps":"1.03867","p50_e2el":5.73562,"p50_tpot":0.00816,"p50_ttft":1.13993,"p90_e2el":28.53768,"p90_tpot":0.01483,"p90_ttft":4.4735,"stdev_isl":"120992.73379","stdev_osl":"1676.20948","median_qps":"1.0","p50_intvty":"124.22360248447205","p90_intvty":67.79661016949153},{"ep":1,"hw":"B300","tp":8,"conc":1,"date":"2026-08-19","num_gpus":8,"scenario":"agentic","framework":"TensorRT LLM","precision":"FP4","tput_per_mw":"1471206.0210526315","tput_per_gpu":2795.29144,"model_display":"MiniMax-M3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1787122800,"serving_config":"agg","total_requests":281,"duration_seconds":"3591.8045","input_tput_per_gpu":2776.148,"total_input_tokens":79771047,"output_tput_per_gpu":19.14344,"total_output_tokens":550076,"avg_output_tokens_expected":"2135.58007","theoretical_cache_hit_rate":"0.98237","stdev_output_tokens_expected":"2860.12685","avg_isl":"283882.72954","avg_osl":"1957.56584","p50_itl":0.00156,"p75_isl":"406130.0","p75_osl":"2671.0","p75_qps":"0.0","p90_isl":"515479.0","p90_itl":0.00235,"p90_osl":"5094.0","p90_qps":"0.0","p95_isl":"543342.0","p95_osl":"6903.0","p95_qps":"1.0","std_qps":"0.27137","mean_qps":"0.07806","p50_e2el":2.91658,"p50_tpot":0.00199,"p50_ttft":1.36134,"p90_e2el":14.62368,"p90_tpot":0.00236,"p90_ttft":1.9276199999999999,"stdev_isl":"166057.47663","stdev_osl":"2596.59967","median_qps":"0.0","p50_intvty":641.025641025641,"p90_intvty":425.531914893617},{"ep":1,"hw":"GB300","tp":8,"conc":48,"date":"2026-09-15","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":4165970.287735849,"tput_per_gpu":8831.85701,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"num_decode_gpu":24,"serving_config":"disagg","total_requests":6734,"num_prefill_gpu":8,"duration_seconds":"3626.98707","input_tput_per_gpu":8772.17718,"total_input_tokens":1018415161,"output_tput_per_gpu":59.67983,"total_output_tokens":6939191,"avg_output_tokens_expected":"1253.33242","theoretical_cache_hit_rate":"0.97418","stdev_output_tokens_expected":"2044.14621","avg_isl":"151192.50713","avg_osl":"1028.6093","p50_itl":0.00667,"p75_isl":"197901.75","p75_osl":"1051.0","p75_qps":"3.0","p90_isl":"325438.0","p90_itl":0.00881,"p90_osl":"2445.2","p90_qps":"4.0","p95_isl":"559697.9","p95_osl":"3958.55","p95_qps":"5.0","std_qps":"1.47143","mean_itl":0.00685,"mean_qps":"1.85687","p50_e2el":5.36897,"p50_tpot":0.00666,"p50_ttft":1.48243,"p90_e2el":20.2504,"p90_tpot":0.0088,"p90_ttft":5.24592,"mean_e2el":9.4097,"mean_tpot":0.00685,"mean_ttft":2.43123,"stdev_isl":"160107.7001","stdev_osl":"2093.09837","median_qps":"2.0","p50_intvty":"149.92503748125938","p90_intvty":"113.50737797956867","mean_intvty":"145.98540145985402"},{"ep":1,"hw":"GB300","tp":8,"conc":1,"date":"2026-09-15","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":"257630.47641509434","tput_per_gpu":546.17661,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789455600,"num_decode_gpu":24,"serving_config":"disagg","total_requests":249,"num_prefill_gpu":8,"duration_seconds":"3608.76888","input_tput_per_gpu":542.09359,"total_input_tokens":62935951,"output_tput_per_gpu":4.08302,"total_output_tokens":471511,"avg_output_tokens_expected":"2062.90361","theoretical_cache_hit_rate":"0.97852","stdev_output_tokens_expected":"2815.50736","avg_isl":"251410.82329","avg_osl":"1893.61446","p50_itl":0.00386,"p75_isl":"382007.0","p75_osl":"2633.0","p75_qps":"0.0","p90_isl":"438767.4","p90_itl":0.00398,"p90_osl":"5065.4","p90_qps":"0.0","p95_isl":"473102.8","p95_osl":"6907.2","p95_qps":"1.0","std_qps":"0.2597","mean_itl":0.00386,"mean_qps":"0.06885","p50_e2el":3.83702,"p50_tpot":0.00385,"p50_ttft":0.89433,"p90_e2el":21.02898,"p90_tpot":0.00397,"p90_ttft":1.48875,"mean_e2el":8.40938,"mean_tpot":0.00385,"mean_ttft":1.01534,"stdev_isl":"147104.79369","stdev_osl":"2529.22031","median_qps":"0.0","p50_intvty":259.0673575129534,"p90_intvty":"251.25628140703517","mean_intvty":259.0673575129534},{"ep":1,"hw":"GB300","tp":8,"conc":48,"date":"2026-09-04","num_gpus":24,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":4527908.745283019,"tput_per_gpu":9599.16654,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788505200,"num_decode_gpu":16,"serving_config":"disagg","total_requests":5515,"num_prefill_gpu":8,"duration_seconds":"3628.68774","input_tput_per_gpu":9533.25389,"total_input_tokens":832451843,"output_tput_per_gpu":65.91265,"total_output_tokens":5767546,"avg_output_tokens_expected":"1261.77469","theoretical_cache_hit_rate":"0.97409","stdev_output_tokens_expected":"1999.56687","avg_isl":"150541.58404","avg_osl":"1040.84025","p50_itl":0.00877,"p75_isl":"194209.5","p75_osl":"1053.5","p75_qps":"2.0","p90_isl":"303472.6","p90_itl":0.01138,"p90_osl":"2467.0","p90_qps":"3.0","p95_isl":"560391.8","p95_osl":"3971.5","p95_qps":"4.0","std_qps":"1.30208","mean_qps":"1.52026","p50_e2el":10.00436,"p50_tpot":0.00876,"p50_ttft":4.191260000000001,"p90_e2el":31.2187,"p90_tpot":0.01137,"p90_ttft":12.46122,"stdev_isl":"155770.67096","stdev_osl":"2187.6001","median_qps":"1.0","p50_intvty":"114.02508551881414","p90_intvty":87.87346221441125},{"ep":1,"hw":"GB300","tp":8,"conc":32,"date":"2026-09-04","num_gpus":32,"scenario":"agentic","decode_ep":1,"decode_tp":8,"framework":"Dynamo vLLM","precision":"FP4","prefill_ep":1,"prefill_tp":8,"tput_per_mw":2865825.537735849,"tput_per_gpu":6075.55014,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788505200,"num_decode_gpu":24,"serving_config":"disagg","total_requests":4174,"num_prefill_gpu":8,"duration_seconds":"3619.00748","input_tput_per_gpu":6038.29036,"total_input_tokens":699341154,"output_tput_per_gpu":37.25978,"total_output_tokens":4314576,"avg_output_tokens_expected":"1414.10574","theoretical_cache_hit_rate":"0.9784","stdev_output_tokens_expected":"2149.36481","avg_isl":"167533.24748","avg_osl":"1033.77791","p50_itl":0.00591,"p75_isl":"221997.25","p75_osl":"1104.5","p75_qps":"2.0","p90_isl":"328299.8","p90_itl":0.00705,"p90_osl":"2658.8","p90_qps":"3.0","p95_isl":"410551.4","p95_osl":"4112.5","p95_qps":"3.0","std_qps":"1.194","mean_qps":"1.15372","p50_e2el":6.04862,"p50_tpot":0.0059,"p50_ttft":2.38038,"p90_e2el":20.66737,"p90_tpot":0.00704,"p90_ttft":7.47517,"stdev_isl":"146509.2636","stdev_osl":"1736.65654","median_qps":"1.0","p50_intvty":169.2047377326565,"p90_intvty":"141.84397163120568"},{"ep":1,"hw":"B300","tp":4,"conc":8,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":5803157.957894737,"tput_per_gpu":11026.00012,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":1571,"duration_seconds":"3606.69525","input_tput_per_gpu":10916.19559,"total_input_tokens":164751812,"output_tput_per_gpu":109.80454,"total_output_tokens":1584213,"avg_output_tokens_expected":"1757.90287","theoretical_cache_hit_rate":"0.9609","stdev_output_tokens_expected":"2829.8459","avg_isl":"100245.42521","avg_osl":"1008.35519","p50_itl":"0.0022742031543320222","p75_isl":"126078.5","p75_osl":"885.5","p75_qps":"1.0","p90_isl":"177754.0","p90_itl":"0.0028993972604137207","p90_osl":"2375.0","p90_qps":"1.0","p95_isl":"207239.5","p95_osl":"4368.0","p95_qps":"2.0","std_qps":"0.70115","mean_itl":0.002384000243932696,"mean_qps":"0.43551","p50_e2el":1.23997,"p50_tpot":0.00227,"p50_ttft":0.32825,"p90_e2el":5.72462,"p90_tpot":0.0029,"p90_ttft":0.62741,"mean_e2el":2.75937,"mean_tpot":0.00238,"mean_ttft":0.38244,"stdev_isl":"52937.12513","stdev_osl":"2149.41537","median_qps":"0.0","p50_intvty":"439.71445475095186","p90_intvty":"344.89927049779584","mean_intvty":"419.46304432854396"},{"ep":1,"hw":"B300","tp":4,"conc":12,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"10176526.489473684","tput_per_gpu":19335.40033,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":2719,"duration_seconds":"3603.88776","input_tput_per_gpu":19165.16917,"total_input_tokens":289528169,"output_tput_per_gpu":170.23116,"total_output_tokens":2454107,"avg_output_tokens_expected":"1577.97617","theoretical_cache_hit_rate":"0.96177","stdev_output_tokens_expected":"2432.6104","avg_isl":"101609.58955","avg_osl":"902.52887","p50_itl":0.002592609343937245,"p75_isl":"134371.5","p75_osl":"801.5","p75_qps":"1.0","p90_isl":"186819.0","p90_itl":"0.0035891963143133464","p90_osl":"2103.6","p90_qps":"2.0","p95_isl":"214968.6","p95_osl":"3897.0","p95_qps":"3.0","std_qps":"0.9631","mean_itl":"0.0027757660699892625","mean_qps":"0.75458","p50_e2el":1.30478,"p50_tpot":0.00259,"p50_ttft":0.36976,"p90_e2el":6.21788,"p90_tpot":0.00359,"p90_ttft":0.68462,"mean_e2el":2.8702,"mean_tpot":0.00278,"mean_ttft":0.43085,"stdev_isl":"56495.16292","stdev_osl":"1933.45758","median_qps":"0.0","p50_intvty":"385.71179354054095","p90_intvty":278.6139047374206,"mean_intvty":360.2609062815831},{"ep":1,"hw":"B300","tp":4,"conc":16,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"13023920.421052631","tput_per_gpu":24745.4488,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":3240,"duration_seconds":"3605.5331","input_tput_per_gpu":24533.84327,"total_input_tokens":370721614,"output_tput_per_gpu":211.60553,"total_output_tokens":3051979,"avg_output_tokens_expected":"1552.8469","theoretical_cache_hit_rate":"0.96517","stdev_output_tokens_expected":"2511.19322","avg_isl":"109206.89383","avg_osl":"941.91451","p50_itl":"0.0027504019656311956","p75_isl":"150562.0","p75_osl":"900.75","p75_qps":"1.0","p90_isl":"200059.7","p90_itl":0.004195845543709283,"p90_osl":"2316.7","p90_qps":"2.0","p95_isl":"226965.15","p95_osl":"3940.15","p95_qps":"3.0","std_qps":"1.08649","mean_itl":"0.0031376436394573936","mean_qps":"0.89897","p50_e2el":1.50775,"p50_tpot":0.00275,"p50_ttft":0.39315,"p90_e2el":7.25466,"p90_tpot":0.004200000000000001,"p90_ttft":0.74247,"mean_e2el":3.23159,"mean_tpot":0.00314,"mean_ttft":0.46309,"stdev_isl":"60669.15138","stdev_osl":"1849.97879","median_qps":"1.0","p50_intvty":"363.58321892433196","p90_intvty":238.3309846806141,"mean_intvty":318.71050855633},{"ep":1,"hw":"B300","tp":4,"conc":4,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"3767147.4210526315","tput_per_gpu":7157.5801,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":1155,"duration_seconds":"3598.07842","input_tput_per_gpu":7082.90622,"total_input_tokens":105738513,"output_tput_per_gpu":74.67389,"total_output_tokens":1074773,"avg_output_tokens_expected":"1649.5918","theoretical_cache_hit_rate":"0.95541","stdev_output_tokens_expected":"2477.97251","avg_isl":"88259.22771","avg_osl":"930.50216","p50_itl":0.0020989904125,"p75_isl":"109362.5","p75_osl":"813.5","p75_qps":"1.0","p90_isl":"185246.2","p90_itl":"0.0026391506839268533","p90_osl":"2449.6","p90_qps":"1.0","p95_isl":"222487.6","p95_osl":"4114.1","p95_qps":"1.0","std_qps":"0.58328","mean_itl":0.002634298392576499,"mean_qps":"0.32082","p50_e2el":1.09207,"p50_tpot":"0.0021000000000000003","p50_ttft":0.29478,"p90_e2el":6.06361,"p90_tpot":0.00264,"p90_ttft":0.63473,"mean_e2el":2.67408,"mean_tpot":0.00263,"mean_ttft":0.35268,"stdev_isl":"58378.99832","stdev_osl":"1816.00005","median_qps":"0.0","p50_intvty":476.4195177094455,"p90_intvty":"378.90977809272977","mean_intvty":379.607717492452},{"ep":1,"hw":"GB300","tp":8,"conc":1,"date":"2026-09-13","num_gpus":8,"scenario":"agentic","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":993871.9292452831,"tput_per_gpu":2107.00849,"model_display":"Kimi-K3","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789282800,"serving_config":"agg","total_requests":244,"duration_seconds":"3599.43044","input_tput_per_gpu":2091.17074,"total_input_tokens":60934943,"output_tput_per_gpu":15.83775,"total_output_tokens":455838,"avg_output_tokens_expected":"2076.06967","theoretical_cache_hit_rate":"0.97783","stdev_output_tokens_expected":"2830.4093","avg_isl":"246787.65984","avg_osl":"1869.07787","p50_itl":0.00386,"p75_isl":"376977.25","p75_osl":"2594.0","p75_qps":"0.0","p90_isl":"431127.3","p90_itl":0.00438,"p90_osl":"5017.8","p90_qps":"0.0","p95_isl":"461260.05","p95_osl":"6823.65","p95_qps":"1.0","std_qps":"0.25554","mean_qps":"0.06765","p50_e2el":4.81452,"p50_tpot":0.00386,"p50_ttft":1.38632,"p90_e2el":21.30106,"p90_tpot":0.00439,"p90_ttft":2.22901,"stdev_isl":"144825.5384","stdev_osl":"2523.3457","median_qps":"0.0","p50_intvty":259.0673575129534,"p90_intvty":"228.31050228310502"},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"1343004.1471676608","tput_per_gpu":2296.5370916567,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":87.8400096466},{"ep":2,"hw":"B300","tp":2,"conc":16,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"23780822.042105265","tput_per_gpu":45183.56188,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":3017,"duration_seconds":"3601.49846","input_tput_per_gpu":44791.13953,"total_input_tokens":339479726,"output_tput_per_gpu":392.42235,"total_output_tokens":2826793,"avg_output_tokens_expected":"1571.17949","theoretical_cache_hit_rate":"0.96367","stdev_output_tokens_expected":"2544.52566","avg_isl":"106937.50083","avg_osl":"936.89659","p50_itl":0.004113998758550597,"p75_isl":"147329.0","p75_osl":"871.0","p75_qps":"1.0","p90_isl":"194035.2","p90_itl":0.007191249954802229,"p90_osl":"2284.8","p90_qps":"2.0","p95_isl":"222548.6","p95_osl":"3987.2","p95_qps":"3.0","std_qps":"1.06428","mean_itl":0.004796879219716759,"mean_qps":"0.83801","p50_e2el":2.23293,"p50_tpot":0.00411,"p50_ttft":0.5444199999999999,"p90_e2el":11.22903,"p90_tpot":0.00719,"p90_ttft":1.17279,"mean_e2el":5.03976,"mean_tpot":0.0048,"mean_ttft":0.7537,"stdev_isl":"59577.76241","stdev_osl":"1883.3948","median_qps":"1.0","p50_intvty":243.0725089358826,"p90_intvty":"139.05788371772726","mean_intvty":"208.46887198861907"},{"ep":2,"hw":"B300","tp":2,"conc":52,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"51508032.978947364","tput_per_gpu":97865.26266,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":7750,"duration_seconds":"3627.10426","input_tput_per_gpu":96859.20955,"total_input_tokens":770888509,"output_tput_per_gpu":1006.0531,"total_output_tokens":7317434,"avg_output_tokens_expected":"1217.36409","theoretical_cache_hit_rate":"0.95588","stdev_output_tokens_expected":"2052.86263","avg_isl":"90662.82632","avg_osl":"941.69277","p50_itl":0.01162813849103139,"p75_isl":"127230.5","p75_osl":"986.0","p75_qps":"3.0","p90_isl":"198470.2","p90_itl":"0.018812909381934886","p90_osl":"2211.2","p90_qps":"5.0","p95_isl":"228701.8","p95_osl":"3595.65","p95_qps":"6.0","std_qps":"2.06879","mean_itl":"0.013132923003802229","mean_qps":"2.13707","p50_e2el":6.08957,"p50_tpot":0.01163,"p50_ttft":0.79944,"p90_e2el":28.74935,"p90_tpot":0.01881,"p90_ttft":2.0046,"mean_e2el":12.78216,"mean_tpot":0.01313,"mean_ttft":1.08521,"stdev_isl":"64765.81813","stdev_osl":"1892.24928","median_qps":"2.0","p50_intvty":85.99828775442303,"p90_intvty":"53.154989464853905","mean_intvty":76.14451099046885},{"ep":2,"hw":"B300","tp":2,"conc":36,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":42449379.41578948,"tput_per_gpu":80653.82089,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":5569,"duration_seconds":"3614.48496","input_tput_per_gpu":79915.56328,"total_input_tokens":623356157,"output_tput_per_gpu":738.2576,"total_output_tokens":5337240,"avg_output_tokens_expected":"1329.39577","theoretical_cache_hit_rate":"0.96226","stdev_output_tokens_expected":"2109.05798","avg_isl":"103736.2548","avg_osl":"958.31244","p50_itl":"0.0070192423430794435","p75_isl":"146467.0","p75_osl":"994.0","p75_qps":"2.0","p90_isl":"207671.8","p90_itl":"0.011682719023231133","p90_osl":"2305.4","p90_qps":"4.0","p95_isl":"232724.6","p95_osl":"3724.4","p95_qps":"4.0","std_qps":"1.52996","mean_itl":0.007745287490289159,"mean_qps":"1.54068","p50_e2el":3.7849,"p50_tpot":0.00702,"p50_ttft":0.68571,"p90_e2el":18.94087,"p90_tpot":0.01168,"p90_ttft":1.67314,"mean_e2el":8.10538,"mean_tpot":0.00775,"mean_ttft":0.88087,"stdev_isl":"65680.76821","stdev_osl":"2069.32508","median_qps":"1.0","p50_intvty":"142.46551851653058","p90_intvty":85.59651208006424,"mean_intvty":129.1107659016885},{"ep":1,"hw":"B300","tp":4,"conc":20,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"15655467.494736843","tput_per_gpu":29745.38824,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":3977,"duration_seconds":"3604.81063","input_tput_per_gpu":29497.68022,"total_input_tokens":449453730,"output_tput_per_gpu":247.70802,"total_output_tokens":3571982,"avg_output_tokens_expected":"1401.57202","theoretical_cache_hit_rate":"0.96515","stdev_output_tokens_expected":"2152.79992","avg_isl":"106948.50515","avg_osl":"898.1046","p50_itl":0.003075104306306306,"p75_isl":"148740.0","p75_osl":"926.0","p75_qps":"2.0","p90_isl":"202551.2","p90_itl":0.004584569935122443,"p90_osl":"2296.0","p90_qps":"3.0","p95_isl":"226462.6","p95_osl":"3749.0","p95_qps":"3.0","std_qps":"1.16287","mean_itl":0.003409186421735259,"mean_qps":"1.10322","p50_e2el":1.59996,"p50_tpot":0.00308,"p50_ttft":0.39235,"p90_e2el":8.22423,"p90_tpot":0.00458,"p90_ttft":0.73643,"mean_e2el":3.42385,"mean_tpot":0.00341,"mean_ttft":0.47828,"stdev_isl":"61883.75424","stdev_osl":"1596.42432","median_qps":"1.0","p50_intvty":"325.19222126847484","p90_intvty":"218.12296772680645","mean_intvty":"293.32511523115977"},{"ep":2,"hw":"B300","tp":2,"conc":20,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"29077069.894736845","tput_per_gpu":55246.4328,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":3702,"duration_seconds":"3608.57823","input_tput_per_gpu":54788.36832,"total_input_tokens":419524193,"output_tput_per_gpu":458.06448,"total_output_tokens":3306143,"avg_output_tokens_expected":"1411.67866","theoretical_cache_hit_rate":"0.96502","stdev_output_tokens_expected":"2185.65475","avg_isl":"106811.51432","avg_osl":"893.00999","p50_itl":0.004958490881053159,"p75_isl":"148092.25","p75_osl":"899.75","p75_qps":"2.0","p90_isl":"201609.5","p90_itl":0.008385636559923592,"p90_osl":"2285.8","p90_qps":"3.0","p95_isl":"227121.1","p95_osl":"3782.65","p95_qps":"3.0","std_qps":"1.19735","mean_itl":0.005890555412052716,"mean_qps":"1.02578","p50_e2el":2.50699,"p50_tpot":0.00496,"p50_ttft":0.5692,"p90_e2el":14.05071,"p90_tpot":0.00839,"p90_ttft":1.44695,"mean_e2el":5.90743,"mean_tpot":0.00589,"mean_ttft":0.85978,"stdev_isl":"61780.73286","stdev_osl":"1621.79099","median_qps":"1.0","p50_intvty":"201.67426420427438","p90_intvty":"119.25153121698274","mean_intvty":"169.76327868063024"},{"ep":1,"hw":"B300","tp":4,"conc":24,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"18415590.178947367","tput_per_gpu":34989.62134,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":4575,"duration_seconds":"3614.073","input_tput_per_gpu":34698.4926,"total_input_tokens":530703191,"output_tput_per_gpu":291.12873,"total_output_tokens":4227937,"avg_output_tokens_expected":"1360.62651","theoretical_cache_hit_rate":"0.96533","stdev_output_tokens_expected":"2076.84482","avg_isl":"109641.86689","avg_osl":"919.92175","p50_itl":0.003426525456660426,"p75_isl":"156220.5","p75_osl":"965.5","p75_qps":"2.0","p90_isl":"209025.4","p90_itl":0.005264652004609672,"p90_osl":"2337.8","p90_qps":"3.0","p95_isl":"230825.4","p95_osl":"3785.1","p95_qps":"4.0","std_qps":"1.25597","mean_itl":"0.0037691476823443998","mean_qps":"1.26598","p50_e2el":1.81207,"p50_tpot":0.00343,"p50_ttft":0.41447,"p90_e2el":9.45102,"p90_tpot":0.00526,"p90_ttft":0.78735,"mean_e2el":3.90592,"mean_tpot":0.00377,"mean_ttft":0.50525,"stdev_isl":"64425.39557","stdev_osl":"1598.06181","median_qps":"1.0","p50_intvty":"291.84082028522965","p90_intvty":"189.94607794103217","mean_intvty":"265.31197084270326"},{"ep":1,"hw":"B300","tp":4,"conc":28,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":"20220330.657894738","tput_per_gpu":38418.62825,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":5102,"duration_seconds":"3622.05994","input_tput_per_gpu":38082.49708,"total_input_tokens":585517447,"output_tput_per_gpu":336.13117,"total_output_tokens":4870257,"avg_output_tokens_expected":"1291.00842","theoretical_cache_hit_rate":"0.9647","stdev_output_tokens_expected":"1987.69231","avg_isl":"108143.54155","avg_osl":"954.51764","p50_itl":"0.0037955643289473684","p75_isl":"153655.5","p75_osl":"981.75","p75_qps":"2.0","p90_isl":"209451.5","p90_itl":0.006407550329545455,"p90_osl":"2323.9","p90_qps":"3.0","p95_isl":"231900.95","p95_osl":"3729.3","p95_qps":"4.0","std_qps":"1.36928","mean_itl":0.00427729771054198,"mean_qps":"1.40873","p50_e2el":2.02802,"p50_tpot":0.0038,"p50_ttft":0.42664,"p90_e2el":10.29586,"p90_tpot":0.00641,"p90_ttft":0.81823,"mean_e2el":4.49743,"mean_tpot":0.00428,"mean_ttft":0.53277,"stdev_isl":"65256.85695","stdev_osl":"2088.14076","median_qps":"1.0","p50_intvty":"263.46543315663735","p90_intvty":156.0658829925943,"mean_intvty":233.7924707778382},{"ep":2,"hw":"B300","tp":2,"conc":44,"date":"2026-08-03","num_gpus":4,"scenario":"agentic","framework":"SGLang","precision":"FP4","tput_per_mw":46066513.45789474,"tput_per_gpu":87526.37557,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1785740400,"serving_config":"agg","total_requests":6593,"duration_seconds":"3620.23783","input_tput_per_gpu":86657.23751,"total_input_tokens":682817573,"output_tput_per_gpu":869.13806,"total_output_tokens":6293456,"avg_output_tokens_expected":"1235.66954","theoretical_cache_hit_rate":"0.95674","stdev_output_tokens_expected":"2009.41522","avg_isl":"95167.54421","avg_osl":"954.4931","p50_itl":0.008976013197740114,"p75_isl":"132340.0","p75_osl":"1010.0","p75_qps":"3.0","p90_isl":"200265.8","p90_itl":0.01574717239475909,"p90_osl":"2261.2","p90_qps":"4.0","p95_isl":"228039.2","p95_osl":"3659.2","p95_qps":"5.0","std_qps":"1.75424","mean_itl":"0.010593523066754816","mean_qps":"1.82099","p50_e2el":4.93826,"p50_tpot":0.00898,"p50_ttft":0.72262,"p90_e2el":23.62028,"p90_tpot":0.01575,"p90_ttft":1.78786,"mean_e2el":"10.460509999999998","mean_tpot":0.01059,"mean_ttft":0.94728,"stdev_isl":"64308.34321","stdev_osl":"1965.26578","median_qps":"1.0","p50_intvty":"111.40803583619612","p90_intvty":"63.503464300220394","mean_intvty":94.39730236093558},{"ep":1,"hw":"H200","tp":8,"conc":2,"date":"2026-09-16","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":1640811.890510949,"tput_per_gpu":2247.91229,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789542000,"serving_config":"agg","total_requests":624,"duration_seconds":"3602.218","input_tput_per_gpu":2223.55403,"total_input_tokens":64077811,"output_tput_per_gpu":24.35826,"total_output_tokens":701950,"avg_output_tokens_expected":"1835.61058","theoretical_cache_hit_rate":"0.9498","stdev_output_tokens_expected":"2785.99695","avg_isl":"102688.79968","avg_osl":"1124.91987","p50_itl":0.00333,"p75_isl":"152951.75","p75_osl":"947.75","p75_qps":"0.0","p90_isl":"214055.0","p90_itl":0.00406,"p90_osl":"3266.9","p90_qps":"1.0","p95_isl":"233116.25","p95_osl":"5412.55","p95_qps":"1.0","std_qps":"0.41472","mean_itl":0.00361,"mean_qps":"0.17306","p50_e2el":1.92765,"p50_tpot":0.00333,"p50_ttft":0.57892,"p90_e2el":12.24958,"p90_tpot":0.00406,"p90_ttft":1.42405,"mean_e2el":4.73277,"mean_tpot":0.00361,"mean_ttft":0.80072,"stdev_isl":"67498.51296","stdev_osl":"2206.69328","median_qps":"0.0","p50_intvty":300.3003003003003,"p90_intvty":"246.30541871921181","mean_intvty":277.0083102493075},{"ep":4,"hw":"GB300","tp":4,"conc":704,"date":"2026-08-16","num_gpus":20,"scenario":"agentic","decode_ep":4,"decode_tp":4,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":70936181.69339623,"tput_per_gpu":150384.70519,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"num_decode_gpu":8,"serving_config":"disagg","total_requests":111206,"num_prefill_gpu":12,"duration_seconds":"3629.51319","input_tput_per_gpu":148960.02781,"total_input_tokens":10815369940,"output_tput_per_gpu":1424.67738,"total_output_tokens":103700100,"avg_output_tokens_expected":"1357.37462","theoretical_cache_hit_rate":"0.95739","stdev_output_tokens_expected":"2503.74079","avg_isl":"97234.39117","avg_osl":"929.96517","p50_itl":0.00784,"p75_isl":"141017.75","p75_osl":"927.0","p75_qps":"36.0","p90_isl":"194960.0","p90_itl":0.00835,"p90_osl":"2062.0","p90_qps":"40.0","p95_isl":"221313.75","p95_osl":"3518.0","p95_qps":"42.0","std_qps":"7.52749","mean_qps":"30.64343","p50_e2el":6.75307,"p50_tpot":0.00783,"p50_ttft":2.14243,"p90_e2el":21.6245,"p90_tpot":0.00834,"p90_ttft":8.53155,"stdev_isl":"64297.32234","stdev_osl":"1858.37398","median_qps":"31.0","p50_intvty":"127.55102040816327","p90_intvty":"119.76047904191617"},{"ep":8,"hw":"GB300","tp":8,"conc":96,"date":"2026-08-16","num_gpus":28,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":2,"prefill_tp":2,"tput_per_mw":8477640.53773585,"tput_per_gpu":17972.59794,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"num_decode_gpu":24,"serving_config":"disagg","total_requests":19494,"num_prefill_gpu":4,"duration_seconds":"3608.37726","input_tput_per_gpu":17788.51146,"total_input_tokens":1797512646,"output_tput_per_gpu":184.08649,"total_output_tokens":18599102,"avg_output_tokens_expected":"1281.66013","theoretical_cache_hit_rate":"0.9552","stdev_output_tokens_expected":"2386.96555","avg_isl":"92195.26459","avg_osl":"954.09346","p50_itl":0.00171,"p75_isl":"136432.75","p75_osl":"1008.0","p75_qps":"7.0","p90_isl":"195758.7","p90_itl":0.00202,"p90_osl":"2109.7","p90_qps":"9.0","p95_isl":"225680.35","p95_osl":"3519.4","p95_qps":"11.0","std_qps":"3.01353","mean_qps":"5.40394","p50_e2el":1.71989,"p50_tpot":0.0017,"p50_ttft":0.60258,"p90_e2el":5.95922,"p90_tpot":0.00201,"p90_ttft":2.79016,"stdev_isl":"66822.03942","stdev_osl":"1805.23171","median_qps":"5.0","p50_intvty":584.7953216374269,"p90_intvty":495.049504950495},{"ep":2,"hw":"GB300","tp":2,"conc":52,"date":"2026-08-16","num_gpus":6,"scenario":"agentic","decode_ep":2,"decode_tp":2,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"17878796.372641508","tput_per_gpu":37903.04831,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"num_decode_gpu":4,"serving_config":"disagg","total_requests":8796,"num_prefill_gpu":2,"duration_seconds":"3611.35582","input_tput_per_gpu":37523.27432,"total_input_tokens":813285750,"output_tput_per_gpu":379.77399,"total_output_tokens":8228996,"avg_output_tokens_expected":"1291.63463","theoretical_cache_hit_rate":"0.95775","stdev_output_tokens_expected":"2170.81594","avg_isl":"92435.12631","avg_osl":"935.5382","p50_itl":0.00235,"p75_isl":"131354.25","p75_osl":"976.0","p75_qps":"3.0","p90_isl":"199715.0","p90_itl":0.00254,"p90_osl":"2178.0","p90_qps":"4.0","p95_isl":"227934.25","p95_osl":"3598.25","p95_qps":"5.0","std_qps":"1.53999","mean_qps":"2.43561","p50_e2el":7.44996,"p50_tpot":0.00235,"p50_ttft":5.62512,"p90_e2el":17.0581,"p90_tpot":0.00253,"p90_ttft":14.15859,"stdev_isl":"65166.28334","stdev_osl":"1871.27192","median_qps":"2.0","p50_intvty":425.531914893617,"p90_intvty":"393.70078740157476"},{"ep":8,"hw":"GB300","tp":8,"conc":7,"date":"2026-08-16","num_gpus":60,"scenario":"agentic","decode_ep":8,"decode_tp":8,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":306459.7169811321,"tput_per_gpu":649.6946,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"num_decode_gpu":56,"serving_config":"disagg","total_requests":1465,"num_prefill_gpu":4,"duration_seconds":"3601.12264","input_tput_per_gpu":643.33119,"total_input_tokens":139002871,"output_tput_per_gpu":6.36341,"total_output_tokens":1374925,"avg_output_tokens_expected":"1857.08604","theoretical_cache_hit_rate":"0.95808","stdev_output_tokens_expected":"2691.4587","avg_isl":"94882.5058","avg_osl":"938.51536","p50_itl":0.00138,"p75_isl":"121345.0","p75_osl":"865.0","p75_qps":"1.0","p90_isl":"161141.8","p90_itl":"0.0014199999999999998","p90_osl":"2191.6","p90_qps":"1.0","p95_isl":"191479.6","p95_osl":"3880.2","p95_qps":"2.0","std_qps":"0.64224","mean_qps":"0.40678","p50_e2el":2.33237,"p50_tpot":0.00137,"p50_ttft":1.6179,"p90_e2el":5.99027,"p90_tpot":0.00141,"p90_ttft":3.20313,"stdev_isl":"49839.30203","stdev_osl":"1855.88338","median_qps":"0.0","p50_intvty":724.6376811594204,"p90_intvty":704.2253521126761},{"ep":1,"hw":"H200","tp":8,"conc":4,"date":"2026-09-16","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":"2203952.1313868617","tput_per_gpu":3019.41442,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789542000,"serving_config":"agg","total_requests":989,"duration_seconds":"3598.49796","input_tput_per_gpu":2986.2908,"total_input_tokens":86018986,"output_tput_per_gpu":33.12361,"total_output_tokens":953563,"avg_output_tokens_expected":"1667.41254","theoretical_cache_hit_rate":"0.95456","stdev_output_tokens_expected":"2522.64024","avg_isl":"86925.47118","avg_osl":"964.16785","p50_itl":0.00364,"p75_isl":"110511.0","p75_osl":"868.0","p75_qps":"0.0","p90_isl":"184041.6","p90_itl":0.0054,"p90_osl":"2550.2","p90_qps":"1.0","p95_isl":"222677.8","p95_osl":"4183.2","p95_qps":"1.0","std_qps":"0.56804","mean_itl":0.0041,"mean_qps":"0.27467","p50_e2el":2.05174,"p50_tpot":0.00364,"p50_ttft":0.5208,"p90_e2el":10.01394,"p90_tpot":0.0054,"p90_ttft":1.23176,"mean_e2el":4.27199,"mean_tpot":0.0041,"mean_ttft":0.67218,"stdev_isl":"59049.17487","stdev_osl":"1884.53918","median_qps":"0.0","p50_intvty":"274.72527472527474","p90_intvty":"185.18518518518516","mean_intvty":"243.90243902439022"},{"ep":1,"hw":"H200","tp":8,"conc":24,"date":"2026-09-16","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":9868797.759124089,"tput_per_gpu":13520.25293,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789542000,"serving_config":"agg","total_requests":3611,"duration_seconds":"3615.14562","input_tput_per_gpu":13410.04682,"total_input_tokens":387834176,"output_tput_per_gpu":110.20611,"total_output_tokens":3187289,"avg_output_tokens_expected":"1385.6577","theoretical_cache_hit_rate":"0.96534","stdev_output_tokens_expected":"2136.68584","avg_isl":"107403.53808","avg_osl":"882.66104","p50_itl":0.01163,"p75_isl":"149316.5","p75_osl":"903.0","p75_qps":"2.0","p90_isl":"200712.0","p90_itl":0.0189,"p90_osl":"2220.0","p90_qps":"2.0","p95_isl":"225421.0","p95_osl":"3701.5","p95_qps":"3.0","std_qps":"1.12004","mean_itl":0.01319,"mean_qps":"0.99862","p50_e2el":5.13448,"p50_tpot":0.01163,"p50_ttft":0.93774,"p90_e2el":28.48742,"p90_tpot":0.0189,"p90_ttft":1.93379,"mean_e2el":"12.194930000000001","mean_tpot":0.01319,"mean_ttft":1.19547,"stdev_isl":"61623.41945","stdev_osl":"1597.6024","median_qps":"1.0","p50_intvty":85.98452278589853,"p90_intvty":52.91005291005291,"mean_intvty":75.8150113722517},{"ep":1,"hw":"H200","tp":8,"conc":12,"date":"2026-09-16","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":5641077.554744526,"tput_per_gpu":7728.27625,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789542000,"serving_config":"agg","total_requests":2248,"duration_seconds":"3606.69131","input_tput_per_gpu":7657.27925,"total_input_tokens":234182565,"output_tput_per_gpu":70.997,"total_output_tokens":2048644,"avg_output_tokens_expected":"1653.11383","theoretical_cache_hit_rate":"0.96141","stdev_output_tokens_expected":"2582.07597","avg_isl":"98282.71352","avg_osl":"911.26068","p50_itl":0.00604,"p75_isl":"126475.75","p75_osl":"749.5","p75_qps":"1.0","p90_isl":"175681.3","p90_itl":0.0091,"p90_osl":"2077.0","p90_qps":"2.0","p95_isl":"206907.0","p95_osl":"4044.65","p95_qps":"2.0","std_qps":"0.8767","mean_itl":0.00688,"mean_qps":"0.62347","p50_e2el":2.75436,"p50_tpot":0.00604,"p50_ttft":0.68773,"p90_e2el":14.79648,"p90_tpot":0.00909,"p90_ttft":1.57173,"mean_e2el":6.65696,"mean_tpot":0.00687,"mean_ttft":0.90097,"stdev_isl":"53177.65655","stdev_osl":"2043.2134","median_qps":"0.0","p50_intvty":"165.56291390728475","p90_intvty":"109.89010989010988","mean_intvty":"145.34883720930233"},{"ep":1,"hw":"H200","tp":8,"conc":8,"date":"2026-09-16","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":3609505.153284672,"tput_per_gpu":4945.02206,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789542000,"serving_config":"agg","total_requests":1466,"duration_seconds":"3616.91602","input_tput_per_gpu":4896.46308,"total_input_tokens":148940705,"output_tput_per_gpu":48.55898,"total_output_tokens":1405156,"avg_output_tokens_expected":"1746.03167","theoretical_cache_hit_rate":"0.95939","stdev_output_tokens_expected":"2793.20715","avg_isl":"96644.45157","avg_osl":"958.43793","p50_itl":0.00433,"p75_isl":"122409.0","p75_osl":"822.25","p75_qps":"1.0","p90_isl":"168465.0","p90_itl":0.00609,"p90_osl":"2195.0","p90_qps":"1.0","p95_isl":"195562.75","p95_osl":"4133.0","p95_qps":"2.0","std_qps":"0.68122","mean_itl":0.00515,"mean_qps":"0.40537","p50_e2el":2.26338,"p50_tpot":0.00433,"p50_ttft":0.6209,"p90_e2el":11.10999,"p90_tpot":0.00609,"p90_ttft":1.30476,"mean_e2el":5.12995,"mean_tpot":0.00515,"mean_ttft":0.7752,"stdev_isl":"50222.66109","stdev_osl":"2042.76231","median_qps":"0.0","p50_intvty":230.9468822170901,"p90_intvty":"164.20361247947454","mean_intvty":"194.17475728155338"},{"ep":1,"hw":"H200","tp":8,"conc":10,"date":"2026-09-16","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":4352861.313868614,"tput_per_gpu":5963.42,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789542000,"serving_config":"agg","total_requests":1684,"duration_seconds":"3624.43692","input_tput_per_gpu":5904.71481,"total_input_tokens":171679316,"output_tput_per_gpu":58.70519,"total_output_tokens":1712838,"avg_output_tokens_expected":"1783.83873","theoretical_cache_hit_rate":"0.95922","stdev_output_tokens_expected":"2822.34907","avg_isl":"101668.72387","avg_osl":"1010.79929","p50_itl":0.00518,"p75_isl":"130800.25","p75_osl":"875.25","p75_qps":"1.0","p90_isl":"176870.1","p90_itl":0.00759,"p90_osl":"2375.0","p90_qps":"1.0","p95_isl":"202422.4","p95_osl":"4453.0","p95_qps":"2.0","std_qps":"0.74587","mean_itl":0.00606,"mean_qps":"0.46466","p50_e2el":2.79897,"p50_tpot":0.00518,"p50_ttft":0.67078,"p90_e2el":13.04449,"p90_tpot":0.00759,"p90_ttft":1.47376,"mean_e2el":6.18152,"mean_tpot":0.00606,"mean_ttft":0.85191,"stdev_isl":"52600.39777","stdev_osl":"2165.023","median_qps":"0.0","p50_intvty":"193.05019305019306","p90_intvty":"131.75230566534913","mean_intvty":165.016501650165},{"ep":16,"hw":"GB300","tp":16,"conc":565,"date":"2026-08-16","num_gpus":28,"scenario":"agentic","decode_ep":16,"decode_tp":16,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":4,"prefill_tp":4,"tput_per_mw":47272647.16509434,"tput_per_gpu":100218.01199,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"num_decode_gpu":16,"serving_config":"disagg","total_requests":102517,"num_prefill_gpu":12,"duration_seconds":"3626.09708","input_tput_per_gpu":99275.15727,"total_input_tokens":10080239708,"output_tput_per_gpu":942.85472,"total_output_tokens":95789032,"avg_output_tokens_expected":"1365.68611","theoretical_cache_hit_rate":"0.95782","stdev_output_tokens_expected":"2533.06148","avg_isl":"98320.06431","avg_osl":"933.78383","p50_itl":0.00381,"p75_isl":"142338.0","p75_osl":"918.0","p75_qps":"33.0","p90_isl":"197469.4","p90_itl":0.00422,"p90_osl":"2062.4","p90_qps":"37.0","p95_isl":"223685.0","p95_osl":"3543.0","p95_qps":"40.0","std_qps":"7.30749","mean_qps":"28.28","p50_e2el":3.68467,"p50_tpot":0.00381,"p50_ttft":1.30808,"p90_e2el":12.20636,"p90_tpot":0.004200000000000001,"p90_ttft":5.92236,"stdev_isl":"64811.64264","stdev_osl":"1881.60237","median_qps":"28.0","p50_intvty":262.4671916010499,"p90_intvty":"236.96682464454977"},{"ep":2,"hw":"GB300","tp":2,"conc":44,"date":"2026-08-16","num_gpus":3,"scenario":"agentic","decode_ep":2,"decode_tp":2,"framework":"Dynamo TensorRT LLM","precision":"FP4","prefill_ep":1,"prefill_tp":1,"tput_per_mw":"28009542.952830188","tput_per_gpu":59380.23106,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1786863600,"num_decode_gpu":2,"serving_config":"disagg","total_requests":6671,"num_prefill_gpu":1,"duration_seconds":"3615.59538","input_tput_per_gpu":58784.80944,"total_input_tokens":638032615,"output_tput_per_gpu":595.42162,"total_output_tokens":6467683,"avg_output_tokens_expected":"1295.14387","theoretical_cache_hit_rate":"0.95713","stdev_output_tokens_expected":"2070.08038","avg_isl":"95581.81037","avg_osl":"968.13236","p50_itl":0.00283,"p75_isl":"133456.5","p75_osl":"1015.0","p75_qps":"3.0","p90_isl":"200349.0","p90_itl":0.00305,"p90_osl":"2296.0","p90_qps":"4.0","p95_isl":"227987.5","p95_osl":"3709.0","p95_qps":"4.0","std_qps":"1.36233","mean_qps":"1.84509","p50_e2el":9.93525,"p50_tpot":0.00282,"p50_ttft":7.72895,"p90_e2el":17.72812,"p90_tpot":0.00304,"p90_ttft":13.90349,"stdev_isl":"64408.10624","stdev_osl":"1993.34733","median_qps":"2.0","p50_intvty":"353.35689045936397","p90_intvty":"327.86885245901635"},{"ep":1,"hw":"H200","tp":8,"conc":16,"date":"2026-09-08","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":7196054.708029198,"tput_per_gpu":9858.59495,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788850800,"serving_config":"agg","total_requests":2685,"duration_seconds":"3608.2304","input_tput_per_gpu":9771.85631,"total_input_tokens":282072872,"output_tput_per_gpu":86.73864,"total_output_tokens":2503784,"avg_output_tokens_expected":"1552.37745","theoretical_cache_hit_rate":"0.96335","stdev_output_tokens_expected":"2494.57336","avg_isl":"105055.07337","avg_osl":"932.50801","p50_itl":0.00729,"p75_isl":"141777.0","p75_osl":"835.0","p75_qps":"1.0","p90_isl":"192352.2","p90_itl":0.01291,"p90_osl":"2250.6","p90_qps":"2.0","p95_isl":"220974.2","p95_osl":"4027.2","p95_qps":"3.0","std_qps":"0.99398","mean_qps":"0.74452","p50_e2el":3.65194,"p50_tpot":0.00729,"p50_ttft":0.77203,"p90_e2el":18.65113,"p90_tpot":0.01291,"p90_ttft":1.64455,"stdev_isl":"58421.65413","stdev_osl":"1896.07862","median_qps":"0.0","p50_intvty":"137.17421124828533","p90_intvty":77.45933384972889},{"ep":1,"hw":"H200","tp":8,"conc":2,"date":"2026-09-08","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":"1593249.6788321168","tput_per_gpu":2182.75206,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788850800,"serving_config":"agg","total_requests":616,"duration_seconds":"3598.42255","input_tput_per_gpu":2158.66919,"total_input_tokens":62142431,"output_tput_per_gpu":24.08288,"total_output_tokens":693283,"avg_output_tokens_expected":"1852.3263","theoretical_cache_hit_rate":"0.94852","stdev_output_tokens_expected":"2799.62503","avg_isl":"100880.56981","avg_osl":"1125.45942","p50_itl":0.00326,"p75_isl":"149812.0","p75_osl":"944.25","p75_qps":"0.0","p90_isl":"207666.5","p90_itl":0.00405,"p90_osl":"3286.5","p90_qps":"1.0","p95_isl":"228483.5","p95_osl":"5475.25","p95_qps":"1.0","std_qps":"0.41712","mean_qps":"0.17098","p50_e2el":1.84935,"p50_tpot":0.00326,"p50_ttft":0.54075,"p90_e2el":11.93107,"p90_tpot":0.00405,"p90_ttft":1.3315,"stdev_isl":"66029.60894","stdev_osl":"2219.94071","median_qps":"0.0","p50_intvty":306.7484662576687,"p90_intvty":246.9135802469136},{"ep":1,"hw":"H200","tp":8,"conc":4,"date":"2026-09-08","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":"2173024.3357664235","tput_per_gpu":2977.04334,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788850800,"serving_config":"agg","total_requests":978,"duration_seconds":"3600.15359","input_tput_per_gpu":2945.18883,"total_input_tokens":84825057,"output_tput_per_gpu":31.85451,"total_output_tokens":917449,"avg_output_tokens_expected":"1634.05419","theoretical_cache_hit_rate":"0.95434","stdev_output_tokens_expected":"2487.93946","avg_isl":"86733.18712","avg_osl":"938.08691","p50_itl":0.00356,"p75_isl":"110456.25","p75_osl":"840.0","p75_qps":"0.0","p90_isl":"184931.9","p90_itl":0.00507,"p90_osl":"2440.3","p90_qps":"1.0","p95_isl":"222721.75","p95_osl":"4034.8","p95_qps":"1.0","std_qps":"0.56122","mean_qps":"0.27126","p50_e2el":2.03604,"p50_tpot":0.00356,"p50_ttft":0.51682,"p90_e2el":9.67853,"p90_tpot":0.00507,"p90_ttft":1.18362,"stdev_isl":"59101.38816","stdev_osl":"1850.39571","median_qps":"0.0","p50_intvty":280.8988764044944,"p90_intvty":"197.23865877712032"},{"ep":1,"hw":"H200","tp":8,"conc":24,"date":"2026-09-08","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":9869667.766423358,"tput_per_gpu":13521.44484,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788850800,"serving_config":"agg","total_requests":3608,"duration_seconds":"3610.39593","input_tput_per_gpu":13411.28186,"total_input_tokens":387360299,"output_tput_per_gpu":110.16299,"total_output_tokens":3181856,"avg_output_tokens_expected":"1385.36064","theoretical_cache_hit_rate":"0.96535","stdev_output_tokens_expected":"2137.33415","avg_isl":"107361.50194","avg_osl":"881.88914","p50_itl":0.01155,"p75_isl":"149227.0","p75_osl":"900.75","p75_qps":"2.0","p90_isl":"200829.3","p90_itl":0.0191,"p90_osl":"2217.9","p90_qps":"3.0","p95_isl":"225443.9","p95_osl":"3703.45","p95_qps":"3.0","std_qps":"1.15178","mean_qps":"0.99917","p50_e2el":5.06072,"p50_tpot":0.01155,"p50_ttft":0.92644,"p90_e2el":28.36973,"p90_tpot":0.0191,"p90_ttft":1.96765,"stdev_isl":"61630.46763","stdev_osl":"1597.78741","median_qps":"1.0","p50_intvty":86.58008658008659,"p90_intvty":52.35602094240838},{"ep":1,"hw":"H200","tp":8,"conc":8,"date":"2026-09-08","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":3606124.094890511,"tput_per_gpu":4940.39001,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1788850800,"serving_config":"agg","total_requests":1463,"duration_seconds":"3612.88617","input_tput_per_gpu":4891.92816,"total_input_tokens":141391837,"output_tput_per_gpu":48.46184,"total_output_tokens":1400697,"avg_output_tokens_expected":"1748.20106","theoretical_cache_hit_rate":"0.95935","stdev_output_tokens_expected":"2795.88246","avg_isl":"96645.13807","avg_osl":"957.41422","p50_itl":0.00437,"p75_isl":"122372.5","p75_osl":"821.5","p75_qps":"1.0","p90_isl":"168289.4","p90_itl":0.00608,"p90_osl":"2191.8","p90_qps":"1.0","p95_isl":"195619.9","p95_osl":"4141.4","p95_qps":"2.0","std_qps":"0.68292","mean_qps":"0.40487","p50_e2el":2.252,"p50_tpot":0.00437,"p50_ttft":0.60413,"p90_e2el":10.4689,"p90_tpot":0.00608,"p90_ttft":1.26058,"stdev_isl":"50170.87712","stdev_osl":"2043.18132","median_qps":"0.0","p50_intvty":228.8329519450801,"p90_intvty":164.4736842105263},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":512,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":5520810.32491462,"tput_per_gpu":9440.585655604,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":26.3495185538},{"hw":"B200","tp":2,"isl":1024,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":2315016.867045497,"tput_per_gpu":3958.6788426478,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":63.0975202207},{"ep":1,"hw":"H200","tp":8,"conc":4,"date":"2026-09-10","num_gpus":8,"scenario":"agentic","framework":"SGLang","precision":"FP8","tput_per_mw":"2111452.7372262776","tput_per_gpu":2892.69025,"model_display":"Qwen 3.5","spec_decoding":"mtp","benchmark_type":"agentic_traces","date_timestamp":1789023600,"serving_config":"agg","total_requests":946,"duration_seconds":"3609.62192","input_tput_per_gpu":2861.56261,"total_input_tokens":82633273,"output_tput_per_gpu":31.12764,"total_output_tokens":898872,"avg_output_tokens_expected":"1640.34672","theoretical_cache_hit_rate":"0.95436","stdev_output_tokens_expected":"2498.29252","avg_isl":"87350.18288","avg_osl":"950.18182","p50_itl":0.00364,"p75_isl":"110657.75","p75_osl":"850.25","p75_qps":"0.0","p90_isl":"186859.0","p90_itl":0.00476,"p90_osl":"2521.5","p90_qps":"1.0","p95_isl":"222916.25","p95_osl":"4086.25","p95_qps":"1.0","std_qps":"0.51837","mean_qps":"0.26192","p50_e2el":2.01165,"p50_tpot":0.00364,"p50_ttft":0.51184,"p90_e2el":9.85908,"p90_tpot":0.00476,"p90_ttft":1.21201,"stdev_isl":"59697.47314","stdev_osl":"1876.29796","median_qps":"0.0","p50_intvty":"274.72527472527474","p90_intvty":"210.08403361344537"},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":59796.71977372263,"dataset_name":"random","tput_per_gpu":81.92150609,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"14.05918977","input_tput_per_gpu":42.27661835,"total_input_tokens":4755,"output_tput_per_gpu":39.64488774,"total_output_tokens":4459,"tput_requests_per_sec":"0.355639271","p50_e2el":3.225472636,"p50_tpot":0.003131095,"p50_ttft":0.022362628,"p50_intvty":319.3771046},{"hw":"B200","tp":2,"isl":8192,"osl":1024,"conc":128,"date":"2026-05-06","num_gpus":2,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":8538115.107889708,"tput_per_gpu":14600.1768344914,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":35.5503075479},{"hw":"B200","tp":1,"isl":8192,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":5347550.932021872,"tput_per_gpu":9144.3120937574,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":55.0251263419},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"393026.63765713456","tput_per_gpu":672.0755503937,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":130.263866957},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":393512.9575219883,"tput_per_gpu":672.9071573626,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":130.1280332183},{"hw":"B200","tp":1,"isl":1024,"osl":1024,"conc":1024,"date":"2026-05-06","num_gpus":1,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":7222541.40468655,"tput_per_gpu":12350.545802014,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":14.8217520114},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":16,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"110398.13119567253","tput_per_gpu":188.7808043446,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":175.5825758761},{"hw":"B200","tp":4,"isl":1024,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":740161.0973375438,"tput_per_gpu":1265.6754764472,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":107.0933360307},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"1047892.8678832118","dataset_name":"random","tput_per_gpu":1435.613229,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"28.5313615","input_tput_per_gpu":717.8066144,"total_input_tokens":81920,"output_tput_per_gpu":717.8066144,"total_output_tokens":81920,"tput_requests_per_sec":"2.803932087","p50_e2el":5.704254134,"p50_tpot":0.005373723,"p50_ttft":0.206936032,"p50_intvty":186.0907412},{"dp":1,"hw":"H200","pp":1,"tp":8,"isl":1024,"osl":1024,"conc":8,"date":"2026-03-27","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":"363594.04182481754","dataset_name":"random","tput_per_gpu":498.1238373,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"20.55713707","input_tput_per_gpu":249.0619187,"total_input_tokens":40960,"output_tput_per_gpu":249.0619187,"total_output_tokens":40960,"tput_requests_per_sec":"1.94579624","p50_e2el":4.109740406,"p50_tpot":0.003927476,"p50_ttft":0.09193291,"p50_intvty":254.6164795},{"hw":"B200","tp":8,"isl":1024,"osl":1024,"conc":1,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":10087.72376380117,"tput_per_gpu":17.2500076361,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":223.6048624267},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":"135619.27275437958","tput_per_gpu":185.7984036735,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":143.5786861195},{"hw":"H100","tp":8,"isl":1024,"osl":1024,"conc":4,"date":"2026-05-06","num_gpus":8,"sequence":"1K/1K","framework":"vLLM","precision":"FP8","tput_per_mw":73527.47475766424,"tput_per_gpu":100.732640418,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":171.2552190465},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":16,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"420442.13480051095","tput_per_gpu":576.0057246767,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":34.2915503549},{"hw":"B300","tp":2,"isl":1024,"osl":1024,"conc":2048,"date":"2026-06-04","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","tput_per_mw":"15055701.740882918","tput_per_gpu":"28605.833307677545","model_display":"MiniMax-M2.5","spec_decoding":"off","date_timestamp":1780556400,"serving_config":"agg","p50_intvty":"14.700013736252583"},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":512,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":3157859.02279,"tput_per_gpu":4326.2668612223,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":16.0071821281},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":2,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":"129082.45485839417","tput_per_gpu":176.842963156,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":261.1711228229},{"dp":1,"hw":"H200","pp":1,"tp":4,"isl":1024,"osl":1024,"conc":256,"date":"2026-03-27","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":5956798.322627738,"dataset_name":"random","tput_per_gpu":8160.813702,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"80.30571753","input_tput_per_gpu":4080.406851,"total_input_tokens":1310720,"output_tput_per_gpu":4080.406851,"total_output_tokens":1310720,"tput_requests_per_sec":"15.93908926","p50_e2el":"14.846854230000002","p50_tpot":0.0142116,"p50_ttft":0.308387643,"p50_intvty":70.36505491},{"dp":1,"hw":"H200","pp":1,"tp":2,"isl":1024,"osl":1024,"conc":256,"date":"2026-03-27","num_gpus":2,"sequence":"1K/1K","framework":"vLLM","precision":"FP4","spec_length":0,"tput_per_mw":7535603.058394161,"dataset_name":"random","tput_per_gpu":10323.77619,"model_display":"gpt-oss-120b","spec_decoding":"off","date_timestamp":1774594800,"serving_config":"agg","duration_seconds":"126.9612955","input_tput_per_gpu":5161.888097,"total_input_tokens":1310720,"output_tput_per_gpu":5161.888097,"total_output_tokens":1310720,"tput_requests_per_sec":"10.08181269","p50_e2el":25.14870601,"p50_tpot":0.024100329,"p50_ttft":0.494069514,"p50_intvty":41.49320961},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":32,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":5176747.134982993,"tput_per_gpu":7092.1435749267,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":28.0015287379},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":64,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":6345135.088827007,"tput_per_gpu":8692.835071693,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":16.5301969423},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":128,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1072795.4803470804","tput_per_gpu":1469.7298080755,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":11.0832931413},{"hw":"H200","tp":1,"isl":8192,"osl":1024,"conc":2,"date":"2026-05-06","num_gpus":1,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":"1497867.5512747448","tput_per_gpu":2052.0785452464,"model_display":"Nemotron 3 Super 120B","spec_decoding":"off","date_timestamp":1778050800,"serving_config":"agg","p50_intvty":117.249997913},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":16,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":663067.9905259855,"tput_per_gpu":908.4031470206,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":123.7343047309},{"hw":"H100","tp":4,"isl":1024,"osl":1024,"conc":8,"date":"2026-04-09","num_gpus":4,"sequence":"1K/1K","framework":"vLLM","precision":"NVFP4","tput_per_mw":456507.7837735766,"tput_per_gpu":625.4156637698,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":163.3362192954},{"hw":"H200","tp":4,"isl":8192,"osl":1024,"conc":4,"date":"2026-07-04","num_gpus":4,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"1103492.1542856935","tput_per_gpu":1511.7842513714,"model_display":"Qwen 3.5","spec_decoding":"on","date_timestamp":1783148400,"serving_config":"agg","p50_intvty":194.821211892},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":1,"date":"2026-04-09","num_gpus":8,"sequence":"8K/1K","framework":"Dynamo vLLM","precision":"FP4","tput_per_mw":95645.73099642337,"tput_per_gpu":131.0346514651,"model_display":"Kimi K2.5/2.6","spec_decoding":"off","date_timestamp":1775718000,"serving_config":"agg","p50_intvty":124.9045994283},{"hw":"H200","tp":8,"isl":8192,"osl":1024,"conc":2,"date":"2026-07-14","num_gpus":8,"sequence":"8K/1K","framework":"SGLang","precision":"FP4","tput_per_mw":"115098.83136445256","tput_per_gpu":157.6853989693,"model_display":"DeepSeek-V4-Pro","spec_decoding":"off","date_timestamp":1784012400,"serving_config":"agg","p50_intvty":77.9984779377}]