{ "concurrency": { "1": { "aggregate_end_to_end_tokens_per_second": { "maximum": 38.76, "median": 38.682, "minimum": 36.34, "p90": 38.76 }, "per_stream_decode_tokens_per_second": { "maximum": 43.033, "median": 42.979, "minimum": 40.061, "p90": 43.033 }, "per_stream_ttft_seconds": { "maximum": 0.684, "median": 0.679, "minimum": 0.678, "p90": 0.684 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 36.34, "completion_tokens": 256, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.365311, "decode_tokens_per_second": 40.061, "finish_reason": "length", "measured_sse_events": 80, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 180, "reasoning_characters": 1144, "reasoning_sha256": "9f1494357ad612889b2606ad8bba62615d6eb179e104db05ac64d0b01af7250b", "response_tail_seconds": 0.000302, "stream_done_marker": true, "stream_sha256": "9f1494357ad612889b2606ad8bba62615d6eb179e104db05ac64d0b01af7250b", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.043856, "ttft_seconds": 0.678545, "wall_seconds": 7.044157 } ], "wall_seconds": 7.044629 }, { "aggregate_end_to_end_tokens_per_second": 38.76, "completion_tokens": 256, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.925638, "decode_tokens_per_second": 43.033, "finish_reason": "length", "measured_sse_events": 73, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 181, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.000454, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.603968, "ttft_seconds": 0.67833, "wall_seconds": 6.604422 } ], "wall_seconds": 6.604796 }, { "aggregate_end_to_end_tokens_per_second": 38.682, "completion_tokens": 256, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.933117, "decode_tokens_per_second": 42.979, "finish_reason": "length", "measured_sse_events": 73, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1112, "reasoning_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "response_tail_seconds": 0.000451, "stream_done_marker": true, "stream_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.617245, "ttft_seconds": 0.684128, "wall_seconds": 6.617697 } ], "wall_seconds": 6.618105 } ] }, "2": { "aggregate_end_to_end_tokens_per_second": { "maximum": 49.968, "median": 48.931, "minimum": 48.602, "p90": 49.968 }, "per_stream_decode_tokens_per_second": { "maximum": 30.887, "median": 29.313, "minimum": 26.307, "p90": 30.135 }, "per_stream_ttft_seconds": { "maximum": 2.001, "median": 1.333, "minimum": 0.672, "p90": 1.99 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 48.931, "completion_tokens": 512, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 8.985463, "decode_tokens_per_second": 28.379, "finish_reason": "length", "measured_sse_events": 73, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 180, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.00056, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 9.659733, "ttft_seconds": 0.674271, "wall_seconds": 9.660293 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 8.461791, "decode_tokens_per_second": 30.135, "finish_reason": "length", "measured_sse_events": 75, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 179, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.000464, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 10.462615, "ttft_seconds": 2.000824, "wall_seconds": 10.46308 } ], "wall_seconds": 10.463659 }, { "aggregate_end_to_end_tokens_per_second": 49.968, "completion_tokens": 512, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 8.858901, "decode_tokens_per_second": 28.785, "finish_reason": "length", "measured_sse_events": 72, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1112, "reasoning_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "response_tail_seconds": 0.000907, "stream_done_marker": true, "stream_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 9.536784, "ttft_seconds": 0.677884, "wall_seconds": 9.537691 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 8.25596, "decode_tokens_per_second": 30.887, "finish_reason": "length", "measured_sse_events": 72, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 181, "reasoning_characters": 1112, "reasoning_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "response_tail_seconds": 0.000505, "stream_done_marker": true, "stream_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 10.245612, "ttft_seconds": 1.989651, "wall_seconds": 10.246116 } ], "wall_seconds": 10.246611 }, { "aggregate_end_to_end_tokens_per_second": 48.602, "completion_tokens": 512, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 9.693371, "decode_tokens_per_second": 26.307, "finish_reason": "length", "measured_sse_events": 80, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 179, "reasoning_characters": 1160, "reasoning_sha256": "55dbaaa0352c32e86abe49d738718a3d0ba29144c2306ee8ef61eeeacd68751f", "response_tail_seconds": 0.00082, "stream_done_marker": true, "stream_sha256": "55dbaaa0352c32e86abe49d738718a3d0ba29144c2306ee8ef61eeeacd68751f", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 10.365803, "ttft_seconds": 0.672432, "wall_seconds": 10.366623 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 8.544884, "decode_tokens_per_second": 29.842, "finish_reason": "length", "measured_sse_events": 74, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1175, "reasoning_sha256": "05489e7a0d1ea6f34e27fffec2932ab4456e3fa60a4807c004ae77bb5f42f3aa", "response_tail_seconds": 0.000484, "stream_done_marker": true, "stream_sha256": "05489e7a0d1ea6f34e27fffec2932ab4456e3fa60a4807c004ae77bb5f42f3aa", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 10.533505, "ttft_seconds": 1.988621, "wall_seconds": 10.533989 } ], "wall_seconds": 10.534439 } ] }, "4": { "aggregate_end_to_end_tokens_per_second": { "maximum": 63.843, "median": 63.265, "minimum": 61.57, "p90": 63.843 }, "per_stream_decode_tokens_per_second": { "maximum": 22.327, "median": 20.069, "minimum": 17.793, "p90": 22.11 }, "per_stream_ttft_seconds": { "maximum": 4.651, "median": 2.646, "minimum": 0.683, "p90": 4.645 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 63.265, "completion_tokens": 1024, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 12.400714, "decode_tokens_per_second": 20.563, "finish_reason": "length", "measured_sse_events": 81, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1112, "reasoning_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "response_tail_seconds": 4.5e-05, "stream_done_marker": true, "stream_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 15.704409, "ttft_seconds": 3.303695, "wall_seconds": 15.704455 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 11.533499, "decode_tokens_per_second": 22.11, "finish_reason": "length", "measured_sse_events": 82, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 183, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.000636, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 16.184149, "ttft_seconds": 4.65065, "wall_seconds": 16.184785 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 13.158217, "decode_tokens_per_second": 19.38, "finish_reason": "length", "measured_sse_events": 82, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 179, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.000714, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 15.201188, "ttft_seconds": 2.04297, "wall_seconds": 15.201902 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 12.756873, "decode_tokens_per_second": 19.989, "finish_reason": "length", "measured_sse_events": 78, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 181, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.000918, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 13.452635, "ttft_seconds": 0.695763, "wall_seconds": 13.453554 } ], "wall_seconds": 16.185982 }, { "aggregate_end_to_end_tokens_per_second": 63.843, "completion_tokens": 1024, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 11.421056, "decode_tokens_per_second": 22.327, "finish_reason": "length", "measured_sse_events": 80, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 184, "reasoning_characters": 1112, "reasoning_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "response_tail_seconds": 0.000408, "stream_done_marker": true, "stream_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 16.037918, "ttft_seconds": 4.616862, "wall_seconds": 16.038327 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 12.528339, "decode_tokens_per_second": 20.354, "finish_reason": "length", "measured_sse_events": 82, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1125, "reasoning_sha256": "d9122f609c2e202829a5a59a21ad002a97d8c2991bc8b0321266262b96f96a46", "response_tail_seconds": 0.000401, "stream_done_marker": true, "stream_sha256": "d9122f609c2e202829a5a59a21ad002a97d8c2991bc8b0321266262b96f96a46", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 15.777925, "ttft_seconds": 3.249586, "wall_seconds": 15.778326 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 12.986456, "decode_tokens_per_second": 19.636, "finish_reason": "length", "measured_sse_events": 80, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 181, "reasoning_characters": 1115, "reasoning_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "response_tail_seconds": 0.00073, "stream_done_marker": true, "stream_sha256": "444ad1805413e3ddc94b9a34176ad6489e366015500ae54fd7915bf1a4800100", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 13.669043, "ttft_seconds": 0.682587, "wall_seconds": 13.669773 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 13.561347, "decode_tokens_per_second": 18.803, "finish_reason": "length", "measured_sse_events": 85, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 179, "reasoning_characters": 1125, "reasoning_sha256": "d9122f609c2e202829a5a59a21ad002a97d8c2991bc8b0321266262b96f96a46", "response_tail_seconds": 0.000719, "stream_done_marker": true, "stream_sha256": "d9122f609c2e202829a5a59a21ad002a97d8c2991bc8b0321266262b96f96a46", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 15.552654, "ttft_seconds": 1.991307, "wall_seconds": 15.553373 } ], "wall_seconds": 16.039221 }, { "aggregate_end_to_end_tokens_per_second": 61.57, "completion_tokens": 1024, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 14.331142, "decode_tokens_per_second": 17.793, "finish_reason": "length", "measured_sse_events": 91, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 181, "reasoning_characters": 1119, "reasoning_sha256": "e61f96a83bf40870c5e096202f58b02bbaf36b380e6cdf1bc954b836afd75dbc", "response_tail_seconds": 0.000607, "stream_done_marker": true, "stream_sha256": "e61f96a83bf40870c5e096202f58b02bbaf36b380e6cdf1bc954b836afd75dbc", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 16.302999, "ttft_seconds": 1.971857, "wall_seconds": 16.303606 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 12.656638, "decode_tokens_per_second": 20.148, "finish_reason": "length", "measured_sse_events": 82, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1112, "reasoning_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "response_tail_seconds": 0.001055, "stream_done_marker": true, "stream_sha256": "ad4169b9208accdb4fe969575620bd958460f2d6c46800ca73f5a550c32a3f20", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 15.964937, "ttft_seconds": 3.308299, "wall_seconds": 15.965993 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 11.985142, "decode_tokens_per_second": 21.276, "finish_reason": "length", "measured_sse_events": 84, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 182, "reasoning_characters": 1125, "reasoning_sha256": "d9122f609c2e202829a5a59a21ad002a97d8c2991bc8b0321266262b96f96a46", "response_tail_seconds": 0.000404, "stream_done_marker": true, "stream_sha256": "d9122f609c2e202829a5a59a21ad002a97d8c2991bc8b0321266262b96f96a46", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 16.629891, "ttft_seconds": 4.64475, "wall_seconds": 16.630296 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 13.767123, "decode_tokens_per_second": 18.522, "finish_reason": "length", "measured_sse_events": 85, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 180, "reasoning_characters": 1095, "reasoning_sha256": "1a43abaeeceafbbc3378294ef413635d86fe6aba1efbe8d167420f91c6cd7782", "response_tail_seconds": 0.001177, "stream_done_marker": true, "stream_sha256": "1a43abaeeceafbbc3378294ef413635d86fe6aba1efbe8d167420f91c6cd7782", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 14.458795, "ttft_seconds": 0.691673, "wall_seconds": 14.459973 } ], "wall_seconds": 16.631359 } ] } }, "decode": { "code": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 42.778, "median": 41.81, "minimum": 40.718, "p90": 42.778 }, "pooled_decode_tokens_per_second": 41.713, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 7264, "completion_validation": { "valid": true }, "decode_seconds": 176.831766, "decode_tokens_per_second": 41.073, "finish_reason": "stop", "measured_sse_events": 2120, "output": "```go\n// ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype Clock interface {\n\tNow() time.Time\n}\n\ntype systemClock struct{}\n\nfunc (systemClock) Now() time.Time { return time.Now() }\n\ntype Limiter struct {\n\tmu sync.Mutex\n\trate float64\n\tburst int\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\nfunc New(rate float64, burst int, clock Clock) (*Limiter, error) {\n\tif rate <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: rate must be positive\")\n\t}\n\tif burst <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: burst must be positive\")\n\t}\n\tif clock == nil {\n\t\tclock = systemClock{}\n\t}\n\tlimiter := &Limiter{\n\t\trate: rate,\n\t\tburst: burst,\n\t\ttokens: float64(burst),\n\t\tclock: clock,\n\t}\n\tlimiter.last = clock.Now()\n\treturn limiter, nil\n}\n\nfunc (l *Limiter) Allow() bool {\n\treturn l.AllowN(1)\n}\n\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 || n > l.burst {\n\t\treturn false\n\t}\n\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\tif elapsed := now.Sub(l.last).Seconds(); elapsed > 0 {\n\t\tl.tokens += elapsed * l.rate\n\t\tif l.tokens > float64(l.burst) {\n\t\t\tl.tokens = float64(l.burst)\n\t\t}\n\t\tl.last = now\n\t}\n\n\tif l.tokens < float64(n) {\n\t\treturn false\n\t}\n\tl.tokens -= float64(n)\n\treturn true\n}\n```\n\n```go\n// ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tnow time.Time\n}\n\nfunc newFakeClock(now time.Time) *fakeClock {\n\treturn &fakeClock{now: now}\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.now\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.now = c.now.Add(d)\n}\n\ntype step struct {\n\tadvance time.Duration\n\tn int\n\twant bool\n}\n\nfunc TestNewValidation(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 5, false},\n\t\t{\"zero rate\", 0, 5, true},\n\t\t{\"negative rate\", -1, 5, true},\n\t\t{\"zero burst\", 10, 0, true},\n\t\t{\"negative burst\", 10, -1, true},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tclk := newFakeClock(time.Unix(0, 0))\n\t\t\t_, err := New(tc.rate, tc.burst, clk)\n\t\t\tif (err != nil) != tc.wantErr {\n\t\t\t\tt.Fatalf(\"err=%v, wantErr=%v\", err, tc.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\tbase := time.Unix(0, 0)\n\tcases := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tsteps []step\n\t}{\n\t\t{\n\t\t\tname: \"initial burst then empty\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{{0, 2, true}, {0, 1, false}},\n\t\t},\n\t\t{\n\t\t\tname: \"refill after wait\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{{0, 2, true}, {time.Second / 10, 1, true}},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{{0, 2, true}, {time.Millisecond * 50, 1, false}},\n\t\t},\n\t\t{\n\t\t\tname: \"capacity cap\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{{0, 2, true}, {time.Second, 2, true}, {0, 1, false}},\n\t\t},\n\t\t{\n\t\t\tname: \"invalid n\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{{0, 0, false}, {0, 3, false}},\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tclk := newFakeClock(base)\n\t\t\tlimiter, err := New(tc.rate, tc.burst, clk)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, s := range tc.steps {\n\t\t\t\tclk.Advance(s.advance)\n\t\t\t\tif got := limiter.AllowN(s.n); got != s.want {\n\t\t\t\t\tt.Fatalf(\"step %d: AllowN(%d) = %v, want %v\", i, s.n, got, s.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 3400, "output_sha256": "e68421b13285acbae2b2bc0907c39d04cb121f8baf54fd88d2d1489436ffa17f", "prompt_tokens": 179, "reasoning_characters": 22565, "reasoning_sha256": "7fb2cde3a83cc76da098872f5b21fb0bfe1c8327caefca2896cf6692b5f65071", "response_tail_seconds": 0.09015, "run": 1, "stream_done_marker": true, "stream_sha256": "ec000c05f2d880b7fbe3d52fd1f5c5b8428b063bae4f11a92c52055182b0adc5", "time_to_first_visible_seconds": 157.055238, "time_to_last_output_seconds": 177.504144, "ttft_seconds": 0.672378, "wall_seconds": 177.594294 }, { "cached_prompt_tokens": 0, "completion_tokens": 5955, "completion_validation": { "valid": true }, "decode_seconds": 139.185277, "decode_tokens_per_second": 42.778, "finish_reason": "stop", "measured_sse_events": 1638, "output": "```go\n// ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype Clock interface {\n\tNow() time.Time\n}\n\ntype Limiter struct {\n\tmu sync.Mutex\n\trate float64\n\tburst int\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\nfunc New(rate float64, burst int, clock Clock) (*Limiter, error) {\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: nil clock\")\n\t}\n\tif rate <= 0 || math.IsNaN(rate) || math.IsInf(rate, 0) {\n\t\treturn nil, errors.New(\"ratelimit: rate must be positive and finite\")\n\t}\n\tif burst <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: burst must be positive\")\n\t}\n\tnow := clock.Now()\n\treturn &Limiter{\n\t\trate: rate,\n\t\tburst: burst,\n\t\ttokens: float64(burst),\n\t\tlast: now,\n\t\tclock: clock,\n\t}, nil\n}\n\nfunc (l *Limiter) Allow() bool {\n\treturn l.AllowN(1)\n}\n\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n < 0 {\n\t\treturn false\n\t}\n\tif n == 0 {\n\t\treturn true\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last).Seconds()\n\tif elapsed < 0 {\n\t\telapsed = 0\n\t}\n\tl.last = now\n\tl.tokens += elapsed * l.rate\n\tif l.tokens > float64(l.burst) {\n\t\tl.tokens = float64(l.burst)\n\t}\n\tif float64(n) > l.tokens {\n\t\treturn false\n\t}\n\tl.tokens -= float64(n)\n\treturn true\n}\n```\n\n```go\n// ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"math\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tnow time.Time\n}\n\nfunc newFakeClock(now time.Time) *fakeClock {\n\treturn &fakeClock{now: now}\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.now\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.now = c.now.Add(d)\n}\n\nfunc TestNewValidation(t *testing.T) {\n\tclock := newFakeClock(time.Time{})\n\ttests := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 1, 1, clock, false},\n\t\t{\"zero rate\", 0, 1, clock, true},\n\t\t{\"negative rate\", -1, 1, clock, true},\n\t\t{\"nan rate\", math.NaN(), 1, clock, true},\n\t\t{\"inf rate\", math.Inf(1), 1, clock, true},\n\t\t{\"zero burst\", 1, 0, clock, true},\n\t\t{\"negative burst\", 1, -1, clock, true},\n\t\t{\"nil clock\", 1, 1, nil, true},\n\t}\n\tfor _, tc := range tests {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t_, err := New(tc.rate, tc.burst, tc.clock)\n\t\t\tif (err != nil) != tc.wantErr {\n\t\t\t\tt.Fatalf(\"err=%v wantErr=%v\", err, tc.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttype step struct {\n\t\tadvance time.Duration\n\t\tn int\n\t\twant bool\n\t}\n\ttests := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tsteps []step\n\t}{\n\t\t{\n\t\t\tname: \"burst then refill\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{100 * time.Millisecond, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{50 * time.Millisecond, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"cap at burst\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 1, true},\n\t\t\t\t{time.Second, 2, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tests {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tclock := newFakeClock(time.Time{})\n\t\t\tl, err := New(tc.rate, tc.burst, clock)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, s := range tc.steps {\n\t\t\t\tclock.Advance(s.advance)\n\t\t\t\tif got := l.AllowN(s.n); got != s.want {\n\t\t\t\t\tt.Fatalf(\"step %d: got %v want %v\", i, got, s.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 3414, "output_sha256": "5223d36c48fcdbcaa5e63cee30df51190d4280b781ecce6ebfcae59384993a36", "prompt_tokens": 181, "reasoning_characters": 17152, "reasoning_sha256": "7943ca953896680d896a3b1fa740495f5fef3a2cc85494dcbf39957a163dd599", "response_tail_seconds": 0.093762, "run": 2, "stream_done_marker": true, "stream_sha256": "6ec2c32b5a4c8d6ed27e75f963d8d2b9b661f5aadf11ab5eb530687af30f655b", "time_to_first_visible_seconds": 120.054916, "time_to_last_output_seconds": 139.907685, "ttft_seconds": 0.722408, "wall_seconds": 140.001448 }, { "cached_prompt_tokens": 0, "completion_tokens": 5051, "completion_validation": { "valid": true }, "decode_seconds": 124.024649, "decode_tokens_per_second": 40.718, "finish_reason": "stop", "measured_sse_events": 1492, "output": "```go\n// ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype Clock interface {\n\tNow() time.Time\n}\n\ntype Limiter struct {\n\tmu sync.Mutex\n\trate float64\n\tburst int\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\nfunc NewLimiter(rate float64, burst int, clock Clock) (*Limiter, error) {\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: nil clock\")\n\t}\n\tif burst <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: burst must be positive\")\n\t}\n\tif rate <= 0 || math.IsNaN(rate) || math.IsInf(rate, 0) {\n\t\treturn nil, errors.New(\"ratelimit: rate must be positive and finite\")\n\t}\n\tnow := clock.Now()\n\treturn &Limiter{\n\t\trate: rate,\n\t\tburst: burst,\n\t\ttokens: float64(burst),\n\t\tlast: now,\n\t\tclock: clock,\n\t}, nil\n}\n\nfunc New(rate float64, burst int, clock Clock) (*Limiter, error) {\n\treturn NewLimiter(rate, burst, clock)\n}\n\nfunc (l *Limiter) Allow() bool {\n\treturn l.AllowN(1)\n}\n\nfunc (l *Limiter) AllowN(n int) bool {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tif n <= 0 {\n\t\treturn true\n\t}\n\tif n > l.burst {\n\t\treturn false\n\t}\n\n\tnow := l.clock.Now()\n\tif now.Before(l.last) {\n\t\tl.last = now\n\t} else {\n\t\telapsed := now.Sub(l.last).Seconds()\n\t\tl.tokens = math.Min(float64(l.burst), l.tokens+elapsed*l.rate)\n\t\tl.last = now\n\t}\n\n\tif float64(n) <= l.tokens {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"math\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tnow time.Time\n}\n\nfunc newFakeClock(now time.Time) *fakeClock {\n\treturn &fakeClock{now: now}\n}\n\nfunc (c *fakeClock) Now() time.Time { return c.now }\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.now = c.now.Add(d)\n}\n\ntype allowStep struct {\n\tadvance time.Duration\n\tn int\n\twant bool\n}\n\nfunc TestNewLimiterValidation(t *testing.T) {\n\tclk := newFakeClock(time.Unix(0, 0))\n\tcases := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 1, 1, clk, false},\n\t\t{\"zero rate\", 0, 1, clk, true},\n\t\t{\"negative rate\", -1, 1, clk, true},\n\t\t{\"nan rate\", math.NaN(), 1, clk, true},\n\t\t{\"inf rate\", math.Inf(1), 1, clk, true},\n\t\t{\"zero burst\", 1, 0, clk, true},\n\t\t{\"nil clock\", 1, 1, nil, true},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tgot, err := NewLimiter(tc.rate, tc.burst, tc.clock)\n\t\t\tif (err != nil) != tc.wantErr {\n\t\t\t\tt.Fatalf(\"err = %v, wantErr = %v\", err, tc.wantErr)\n\t\t\t}\n\t\t\tif err == nil && got == nil {\n\t\t\t\tt.Fatal(\"expected limiter\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\tstart := time.Unix(100, 0)\n\tcases := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tsteps []allowStep\n\t}{\n\t\t{\n\t\t\tname: \"initial burst and refill\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tsteps: []allowStep{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{time.Second, 1, true},\n\t\t\t\t{500 * time.Millisecond, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"single token refill\",\n\t\t\trate: 10,\n\t\t\tburst: 1,\n\t\t\tsteps: []allowStep{\n\t\t\t\t{0, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{100 * time.Millisecond, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"request larger than burst\",\n\t\t\trate: 1,\n\t\t\tburst: 1,\n\t\t\tsteps: []allowStep{\n\t\t\t\t{0, 2, false},\n\t\t\t\t{time.Second, 1, true},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"insufficient partial refill\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tsteps: []allowStep{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{250 * time.Millisecond, 1, false},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tclk := newFakeClock(start)\n\t\t\tlim, err := NewLimiter(tc.rate, tc.burst, clk)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, step := range tc.steps {\n\t\t\t\tclk.Advance(step.advance)\n\t\t\t\tgot := lim.AllowN(step.n)\n\t\t\t\tif got != step.want {\n\t\t\t\t\tt.Fatalf(\"step %d: AllowN(%d) = %v, want %v\", i, step.n, got, step.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 3776, "output_sha256": "8d95238f5bdf596df50a30be379e87aa70218de62831c73889714771ae451d66", "prompt_tokens": 181, "reasoning_characters": 13889, "reasoning_sha256": "e13a6c87241aaee6f74ff81c0b854ac117e0d5b9fa814569a118bca466636de1", "response_tail_seconds": 0.003472, "run": 3, "stream_done_marker": true, "stream_sha256": "0735e3c172d380d2261e96d2664e94acc70263dcaa0d34a18e5f7ccb81fbd7c5", "time_to_first_visible_seconds": 101.024082, "time_to_last_output_seconds": 124.721094, "ttft_seconds": 0.696445, "wall_seconds": 124.724565 }, { "cached_prompt_tokens": 0, "completion_tokens": 5901, "completion_validation": { "valid": true }, "decode_seconds": 141.115498, "decode_tokens_per_second": 41.81, "finish_reason": "stop", "measured_sse_events": 1687, "output": "```go\n// ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype Clock interface {\n\tNow() time.Time\n}\n\ntype Limiter struct {\n\tmu sync.Mutex\n\trate float64\n\tburst int\n\tclock Clock\n\ttokens float64\n\tlast time.Time\n}\n\nvar (\n\tErrInvalidRate = errors.New(\"ratelimit: rate must be greater than zero\")\n\tErrInvalidBurst = errors.New(\"ratelimit: burst must be greater than zero\")\n\tErrNilClock = errors.New(\"ratelimit: clock must not be nil\")\n)\n\nfunc New(rate float64, burst int, clock Clock) (*Limiter, error) {\n\tif rate <= 0 {\n\t\treturn nil, ErrInvalidRate\n\t}\n\tif burst <= 0 {\n\t\treturn nil, ErrInvalidBurst\n\t}\n\tif clock == nil {\n\t\treturn nil, ErrNilClock\n\t}\n\tnow := clock.Now()\n\treturn &Limiter{\n\t\trate: rate,\n\t\tburst: burst,\n\t\tclock: clock,\n\t\ttokens: float64(burst),\n\t\tlast: now,\n\t}, nil\n}\n\nfunc (l *Limiter) Allow() bool {\n\treturn l.AllowN(1)\n}\n\nfunc (l *Limiter) AllowN(n int) bool {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\tif now.After(l.last) {\n\t\telapsed := now.Sub(l.last).Seconds()\n\t\tl.tokens = math.Min(float64(l.burst), l.tokens+elapsed*l.rate)\n\t\tl.last = now\n\t}\n\n\tif n <= 0 {\n\t\treturn true\n\t}\n\tif n > l.burst {\n\t\treturn false\n\t}\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tnow time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\treturn c.now\n}\n\nfunc TestNew(t *testing.T) {\n\tclk := &fakeClock{now: time.Unix(0, 0)}\n\tcases := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tclock Clock\n\t\terr error\n\t}{\n\t\t{\"valid\", 1, 1, clk, nil},\n\t\t{\"zero rate\", 0, 1, clk, ErrInvalidRate},\n\t\t{\"negative rate\", -1, 1, clk, ErrInvalidRate},\n\t\t{\"zero burst\", 1, 0, clk, ErrInvalidBurst},\n\t\t{\"negative burst\", 1, -1, clk, ErrInvalidBurst},\n\t\t{\"nil clock\", 1, 1, nil, ErrNilClock},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tgot, err := New(tc.rate, tc.burst, tc.clock)\n\t\t\tif !errors.Is(err, tc.err) {\n\t\t\t\tt.Fatalf(\"err = %v, want %v\", err, tc.err)\n\t\t\t}\n\t\t\tif tc.err == nil && got == nil {\n\t\t\t\tt.Fatal(\"limiter is nil\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttype call struct {\n\t\tat time.Time\n\t\tn int\n\t\twant bool\n\t}\n\tcases := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tcalls []call\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tcalls: []call{\n\t\t\t\t{time.Unix(0, 0), 2, true},\n\t\t\t\t{time.Unix(0, 0), 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"refill\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tcalls: []call{\n\t\t\t\t{time.Unix(0, 0), 2, true},\n\t\t\t\t{time.Unix(1, 0), 1, true},\n\t\t\t\t{time.Unix(1, 0), 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\trate: 2,\n\t\t\tburst: 2,\n\t\t\tcalls: []call{\n\t\t\t\t{time.Unix(0, 0), 2, true},\n\t\t\t\t{time.Unix(0, 250000000), 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"oversized request\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tcalls: []call{\n\t\t\t\t{time.Unix(0, 0), 3, false},\n\t\t\t\t{time.Unix(0, 0), 2, true},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"zero request\",\n\t\t\trate: 1,\n\t\t\tburst: 1,\n\t\t\tcalls: []call{\n\t\t\t\t{time.Unix(0, 0), 0, true},\n\t\t\t\t{time.Unix(0, 0), 1, true},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tclk := &fakeClock{now: tc.calls[0].at}\n\t\t\tlim, err := New(tc.rate, tc.burst, clk)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, c := range tc.calls {\n\t\t\t\tclk.now = c.at\n\t\t\t\tif got := lim.AllowN(c.n); got != c.want {\n\t\t\t\t\tt.Fatalf(\"call %d: got %v, want %v\", i, got, c.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 3523, "output_sha256": "8920433b04e640c6873bb18d7518ad594d40a3d90011abe5011a6b6626b3bb00", "prompt_tokens": 181, "reasoning_characters": 16822, "reasoning_sha256": "24ddbdd5ad6e161a16c1e1fe434ffa23ab2c8f980abd7349fa6de1f57cc58cd8", "response_tail_seconds": 0.09144, "run": 4, "stream_done_marker": true, "stream_sha256": "e4f71e3db6dfa5f83363ccdbb8a6a689f9f26e8b91a5d00d63197da2d9e6573f", "time_to_first_visible_seconds": 119.889615, "time_to_last_output_seconds": 141.794277, "ttft_seconds": 0.678779, "wall_seconds": 141.885717 }, { "cached_prompt_tokens": 0, "completion_tokens": 4431, "completion_validation": { "valid": true }, "decode_seconds": 104.401614, "decode_tokens_per_second": 42.432, "finish_reason": "stop", "measured_sse_events": 1246, "output": "```go\n// ratelimit.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype Clock interface {\n\tNow() time.Time\n}\n\nvar (\n\tErrInvalidRate = errors.New(\"ratelimit: rate must be positive\")\n\tErrInvalidBurst = errors.New(\"ratelimit: burst must be positive\")\n\tErrNilClock = errors.New(\"ratelimit: clock must not be nil\")\n)\n\ntype Limiter struct {\n\tmu sync.Mutex\n\trate float64\n\tburst int\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\nfunc New(rate float64, burst int, clock Clock) (*Limiter, error) {\n\tif rate <= 0 {\n\t\treturn nil, ErrInvalidRate\n\t}\n\tif burst <= 0 {\n\t\treturn nil, ErrInvalidBurst\n\t}\n\tif clock == nil {\n\t\treturn nil, ErrNilClock\n\t}\n\tnow := clock.Now()\n\treturn &Limiter{\n\t\trate: rate,\n\t\tburst: burst,\n\t\ttokens: float64(burst),\n\t\tlast: now,\n\t\tclock: clock,\n\t}, nil\n}\n\nfunc (l *Limiter) Allow() bool {\n\treturn l.AllowN(1)\n}\n\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn true\n\t}\n\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tif n > l.burst {\n\t\treturn false\n\t}\n\n\tnow := l.clock.Now()\n\telapsed := now.Sub(l.last)\n\tif elapsed < 0 {\n\t\telapsed = 0\n\t}\n\n\tl.tokens += l.rate * elapsed.Seconds()\n\tif l.tokens > float64(l.burst) {\n\t\tl.tokens = float64(l.burst)\n\t}\n\tl.last = now\n\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// ratelimit_test.go\npackage ratelimit\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tnow time.Time\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.now\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.now = c.now.Add(d)\n}\n\nfunc TestNew(t *testing.T) {\n\tclk := &fakeClock{now: time.Unix(0, 0)}\n\ttests := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 10, 5, clk, false},\n\t\t{\"zero rate\", 0, 5, clk, true},\n\t\t{\"negative rate\", -1, 5, clk, true},\n\t\t{\"zero burst\", 10, 0, clk, true},\n\t\t{\"nil clock\", 10, 5, nil, true},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := New(tt.rate, tt.burst, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"err=%v wantErr=%v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLimiterAllowN(t *testing.T) {\n\tclk := &fakeClock{now: time.Unix(0, 0)}\n\tl, err := New(10, 5, clk)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\tadvance time.Duration\n\t\tn int\n\t\twant bool\n\t}{\n\t\t{\"burst all\", 0, 5, true},\n\t\t{\"empty\", 0, 1, false},\n\t\t{\"zero n\", 0, 0, true},\n\t\t{\"negative n\", 0, -1, true},\n\t\t{\"too large\", 0, 6, false},\n\t\t{\"after 100ms\", 100 * time.Millisecond, 1, true},\n\t\t{\"no extra token\", 0, 1, false},\n\t\t{\"after 200ms\", 200 * time.Millisecond, 2, true},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tclk.Advance(tt.advance)\n\t\t\tif got := l.AllowN(tt.n); got != tt.want {\n\t\t\t\tt.Errorf(\"AllowN(%d) = %v, want %v\", tt.n, got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 2949, "output_sha256": "f42f0140a3b1706310535679cdc38495a4f97f447ef8d8520171680e3e15a8d6", "prompt_tokens": 179, "reasoning_characters": 12235, "reasoning_sha256": "46a46e2c8253da20db60e439b73423cd330c6b1b912e12b6b258e91b27687edc", "response_tail_seconds": 0.003911, "run": 5, "stream_done_marker": true, "stream_sha256": "7fb5614d9c451f517f580797f0f8c67c517744fbd99c9819f0fad77ef62d4812", "time_to_first_visible_seconds": 87.781317, "time_to_last_output_seconds": 105.106453, "ttft_seconds": 0.704839, "wall_seconds": 105.110365 } ], "time_to_last_output_seconds": { "maximum": 177.504, "median": 139.908, "minimum": 105.106, "p90": 177.504 }, "ttft_seconds": { "maximum": 0.722, "median": 0.696, "minimum": 0.672, "p90": 0.722 }, "wall_seconds": { "maximum": 177.594, "median": 140.001, "minimum": 105.11, "p90": 177.594 } }, "prose": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 45.659, "median": 37.149, "minimum": 29.81, "p90": 45.659 }, "pooled_decode_tokens_per_second": 38.819, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 2005, "completion_validation": { "valid": true }, "decode_seconds": 53.945457, "decode_tokens_per_second": 37.149, "finish_reason": "stop", "measured_sse_events": 669, "output": "MEMORANDUM\n\nTo: Engineering Leadership\nFrom: Platform Engineering\nDate: 14 May 2026\nSubject: Local AI hardware expansion versus utilisation improvements\n\nWe are at a familiar inflection point: our local inference estate is busy enough to feel constrained, yet not clearly saturated in a way that justifies another capital purchase. My recommendation is to spend the next quarter improving utilisation before buying additional hardware, while keeping a narrow, pre-approved purchase path open if the data shows a genuine capacity wall.\n\nThe case for buying is straightforward. More GPUs would reduce queueing, protect latency during bursts, and give us headroom for larger models or new features. But hardware is the most expensive and least reversible way to solve a problem that may be partly self-inflicted. Purchase price is only the visible cost; power, cooling, rack space, networking, maintenance, and the operational attention required to keep another machine healthy all compound quietly. A server that is only busy at peak can look like a bargain in a spreadsheet and still become a liability in practice.\n\nImproving utilisation attacks the same symptoms from the other direction. Better batching, request routing, model selection, caching, quantisation, and scheduling can turn idle or fragmented capacity into useful throughput without changing our physical footprint. These changes are comparatively cheap, can be rolled back if they harm quality, and often reveal that our apparent shortage is really a mismatch between workload shape and resource allocation. If a small number of long-running jobs are monopolising accelerators while short requests wait, adding hardware merely papers over a scheduling problem.\n\nLatency deserves particular care because it is where this decision will be felt by users. Buying capacity can reduce average wait times, but it does not automatically fix tail latency caused by poor queue management or uneven load. Utilisation work can improve both average and worst-case behaviour by smoothing demand, prioritising interactive requests, and preventing one tenant from starving another. There is a risk that aggressive batching increases latency for individual requests, so we should measure p95 and p99 alongside throughput rather than celebrating throughput alone.\n\nReliability cuts in a similar direction. Every additional machine expands the failure surface and the number of components that can degrade silently. A larger estate also demands more monitoring, more runbooks, and more on-call familiarity. By contrast, better utilisation can improve reliability by making degradation graceful: if we can shed load, route around a sick node, or fall back to a smaller model, we become less dependent on any single box. The caveat is that software changes can introduce their own regressions, which is why they must be staged and observed rather than deployed as a single sweeping rewrite.\n\nThe opportunity cost is the factor most likely to be underestimated. Engineering time spent tuning inference is time not spent on product features, evaluation, or customer-facing improvements. Yet buying hardware consumes budget that could fund those same priorities, and it locks us into a fixed capacity decision before we fully understand our growth curve. A modest, time-boxed utilisation programme is a reversible bet; a hardware purchase is a commitment that will shape our roadmap for years.\n\nUncertainty remains substantial. We do not yet know whether demand will grow linearly or in steps, whether future models will require materially more memory, or whether cloud bursting would be cheaper than owning more local capacity. Our current telemetry is also too coarse to distinguish true saturation from inefficient scheduling. For that reason, I propose a six-week utilisation sprint with explicit success criteria: reduce p95 latency, increase useful accelerator occupancy, and document the remaining bottleneck. If those metrics do not move, or if they move but capacity is still exhausted at peak, we should buy a narrowly specified machine rather than a speculative expansion. This keeps the decision evidence-led, preserves optionality, and avoids turning a software problem into a capital expense.", "output_characters": 4247, "output_sha256": "08d0560c41b8e459f17f153dbe8f63c1d30b9023445579d93a6d610a28cceca5", "prompt_tokens": 169, "reasoning_characters": 6378, "reasoning_sha256": "badebafcd90a79e7bf6ef4bd029325e995310c8bc94116b808fa92191c41da52", "response_tail_seconds": 0.002227, "run": 1, "stream_done_marker": true, "stream_sha256": "4982d822374db43a919c4bcd399042c955a5776452a8932fe3205bac992813ce", "time_to_first_visible_seconds": 41.906119, "time_to_last_output_seconds": 54.616013, "ttft_seconds": 0.670556, "wall_seconds": 54.618241 }, { "cached_prompt_tokens": 0, "completion_tokens": 4857, "completion_validation": { "valid": true }, "decode_seconds": 106.35347, "decode_tokens_per_second": 45.659, "finish_reason": "stop", "measured_sse_events": 1252, "output": "To: Engineering Team \nFrom: Platform Engineering \nSubject: Whether to buy more local AI hardware or improve utilization\n\nWe are at a familiar inflection point: our local AI estate is busy enough that the team is tempted to buy more hardware, yet not so saturated that the case is obvious. My recommendation is to spend the next quarter improving utilization of the machines we already own, and to treat any purchase as a later, evidence-driven decision rather than a reflex. This is not a claim that our current capacity is ample forever; it is a claim that the cheapest and least risky gains are still available inside our existing footprint, and that we should exhaust them before committing capital.\n\nThe cost argument is stronger than it first appears. A new accelerator is not merely its invoice price; it brings power, cooling, rack space, networking, spares, and the quiet tax of maintenance and depreciation. Those costs are fixed whether the machine is busy or idle, which means a purchase made to solve a temporary peak can become an expensive ornament during normal weeks. By contrast, utilization work\u2014better batching, queue shaping, model placement, cache reuse, and scheduling discipline\u2014usually costs engineering time rather than capital, and its benefits compound across every workload we run. If we can raise effective throughput without adding a single watt of new power draw, we have improved both unit economics and headroom, while preserving budget for the surprises that always arrive.\n\nLatency and reliability also favor patience, though in a more conditional way. Buying hardware can reduce queueing delay, but only if compute is genuinely the bottleneck and not an artifact of poor scheduling, oversized models, or uneven traffic. If requests are waiting because jobs are poorly packed or because one tenant monopolizes a device, a new box may simply inherit the same disorder at higher cost. Reliability cuts similarly: every additional machine expands the failure surface, the spare-parts burden, and the operational knowledge required to keep the fleet healthy. A smaller, better-understood estate is easier to monitor, easier to repair, and easier to reason about during incidents. Improving utilization can make latency more predictable by smoothing contention and reducing the variance that comes from ad hoc placement, especially in the tail where user experience is actually decided.\n\nThere is, of course, an opportunity cost to this path. Engineering hours spent on schedulers, profilers, and capacity experiments are hours not spent on product features, model quality, or customer-facing improvements. That cost is real and should be budgeted explicitly rather than hidden inside vague platform work. Yet the alternative has its own opportunity cost: capital spent on hardware is capital that cannot fund cloud burst capacity, better evaluation tooling, or the next model iteration. Hardware is also comparatively irreversible. Utilization changes can be rolled back, tuned, or abandoned with little residue; a purchased machine is a long-lived commitment with weak resale value and a planning error that can haunt us for years. In an environment where model sizes, request patterns, and user growth are all moving, reversibility is not a luxury but a risk-management tool.\n\nThe honest caveat is that utilization work has diminishing returns, and we do not yet know where our ceiling is. If, after a quarter of focused measurement and tuning, p95 latency still breaches our service target during ordinary load, or if queue depth grows faster than traffic, then the evidence will support a targeted purchase. Until then, we should instrument first, optimize second, and buy third. That sequence keeps our options open while forcing the decision to be driven by observed saturation rather than anxiety.", "output_characters": 3837, "output_sha256": "2af2625fbe0ce492018a856be44d4e11a04f781800ec10e1987abde602aaced1", "prompt_tokens": 171, "reasoning_characters": 16045, "reasoning_sha256": "b0a3df14bed272fd38e358486b88da2db5103172ea6106acda9cd6f57b97c7d3", "response_tail_seconds": 0.003396, "run": 2, "stream_done_marker": true, "stream_sha256": "e9925ef29f51e3fe24982460c38bcd350f1c264a168b4e414d757a4c4d79ce48", "time_to_first_visible_seconds": 94.969227, "time_to_last_output_seconds": 107.041888, "ttft_seconds": 0.688418, "wall_seconds": 107.045284 }, { "cached_prompt_tokens": 0, "completion_tokens": 3264, "completion_validation": { "valid": true }, "decode_seconds": 79.543543, "decode_tokens_per_second": 41.022, "finish_reason": "stop", "measured_sse_events": 953, "output": "To: Engineering Team\nFrom: Engineering Lead\nDate: Current quarter\nSubject: Buy more local AI hardware or improve utilization of what we own\n\nWe are at a decision point: whether to expand our local AI hardware footprint or extract more value from the machines we already own. My recommendation is to prioritize utilization improvements for the next quarter, while keeping a narrowly scoped hardware purchase as a contingent option rather than the default path. This is not a rejection of growth; it is a sequencing decision based on what we can learn cheaply and reverse easily.\n\nThe immediate cost case favors utilization. New hardware carries not only purchase price but rack space, power, cooling, maintenance, and the hidden tax of integration. Those costs are committed before we know whether our current bottleneck is truly capacity or merely uneven scheduling. If we improve batching, queue fairness, model placement, and idle-time recovery, we may discover that our existing estate has substantial headroom. That would convert a capital decision into an operational one, preserving cash and avoiding depreciation on equipment that could be stranded by changing model sizes or vendor roadmaps.\n\nLatency is the strongest argument for buying. Local inference can reduce round-trip time, and for interactive workloads that matters. Yet latency is not only a function of hardware count; it is also a function of contention, cold starts, and poor routing. A second GPU that sits behind an inefficient scheduler may not improve user-facing response times as much as better queue management on the first. We should measure p95 and p99 latency by workload class before assuming that capacity is the binding constraint. If utilization work reduces tail latency materially, we will have answered the question without spending. If it does not, the evidence for hardware becomes much stronger and more specific.\n\nReliability cuts both ways. More machines can improve fault tolerance, but they also expand the surface area for failures, driver issues, thermal events, and configuration drift. Our current reliability problem, if we have one, may be operational rather than physical: a single overloaded node can create cascading timeouts even when aggregate capacity looks adequate. Improving utilization gives us better observability into saturation points and failure modes. It also lets us test graceful degradation, fallback routing, and load shedding under controlled conditions. Buying first would add complexity before we understand the failure envelope of what we already have.\n\nOpportunity cost is where this decision becomes strategic. Engineering time spent tuning schedulers, profiling kernels, and rewriting inference paths is time not spent on product features, evaluation harnesses, or customer-facing improvements. That cost is real and should not be romanticized. However, the same tuning work usually pays dividends beyond this decision: it improves our understanding of model behavior, exposes wasteful patterns, and creates operational competence that will matter regardless of fleet size. Hardware purchases, by contrast, consume budget that could fund experiments, data acquisition, or a later, better-informed expansion. If we buy now, we risk optimizing for today's workload shape while our roadmap is still moving.\n\nReversibility should weigh heavily in a small team. Utilization improvements are largely reversible: we can revert a scheduler change, relax a batching policy, or abandon an optimization if it proves fragile. Hardware is not. Resale value is uncertain, lead times are long, and sunk costs create psychological pressure to justify the purchase. There is also a softer irreversibility: once we own more capacity, organizational habits form around it, and future decisions may assume that baseline. Starting with software keeps our options open and forces us to earn the right to spend.\n\nUncertainty remains substantial. We do not yet know whether our peak load is bursty or sustained, whether model sizes will grow faster than our tuning can compensate, or whether a future customer commitment will require dedicated capacity. It is possible that utilization work will reveal a hard ceiling that only hardware can raise. In that case, we should buy, but with a precise specification derived from measured saturation rather than intuition. We should also define in advance what evidence would trigger purchase: for example, sustained p99 latency above target after tuning, or queue wait times that cannot be reduced without unacceptable batching delays.\n\nFor the next quarter, I propose we treat utilization as the primary investment, with a documented checkpoint at the end. We should instrument current nodes, establish latency and throughput baselines, and run a small set of controlled experiments on scheduling and batching. If those experiments fail to move the metrics, we will have a defensible, data-backed case for hardware. If they succeed, we will have improved the system, preserved flexibility, and learned something durable about our own workload. Either outcome is better than buying capacity we have not yet proven we need.", "output_characters": 5171, "output_sha256": "ade0a056ef9f53a1c52092b73876a2c1baf6edbc1446a20583cea4dc622cf88e", "prompt_tokens": 175, "reasoning_characters": 11853, "reasoning_sha256": "215a23f54c176abb628780669e58c8ea0caf243421a0909365582932f1f18ffc", "response_tail_seconds": 0.002962, "run": 3, "stream_done_marker": true, "stream_sha256": "bbc87fc62ce517302bbce4c40f00f46552e81ba3a386fe6f07b17485d4327da2", "time_to_first_visible_seconds": 64.27128, "time_to_last_output_seconds": 80.225762, "ttft_seconds": 0.682219, "wall_seconds": 80.228723 }, { "cached_prompt_tokens": 0, "completion_tokens": 1807, "completion_validation": { "valid": true }, "decode_seconds": 60.582965, "decode_tokens_per_second": 29.81, "finish_reason": "stop", "measured_sse_events": 795, "output": "To: AI Infrastructure Working Group \nFrom: Engineering Lead \nDate: 18 May 2026 \nSubject: Whether to expand local AI hardware or improve utilisation first\n\nMy recommendation is to defer any new hardware purchase for one quarter and first invest in measuring and improving utilisation of the machines we already own. That path is cheaper to test, easier to reverse, and more likely to reveal whether our bottleneck is genuinely capacity or simply uneven use. If, after a focused improvement cycle, latency and saturation remain unacceptable, we can buy with evidence rather than instinct.\n\nCost is the most obvious difference, but it is not only the purchase price. New local hardware carries capital expense, power, cooling, rack space, maintenance, and the quieter cost of operational attention. For a small team, those recurring burdens can be larger than the invoice. Improving utilisation costs engineering time, yet it usually avoids long-lived commitments and can be funded from existing capacity. The danger is that we treat hardware as a shortcut around software work. If our current machines are idle during some periods and overloaded during others, buying more only spreads the same inefficiency across a larger estate. We should establish a utilisation baseline before assuming scarcity.\n\nLatency is where the decision becomes concrete. Additional hardware can reduce queueing when demand genuinely exceeds supply, especially for large models or long contexts. However, latency is also shaped by scheduling, batching, model placement, cache behaviour, and request routing. If those are weak, new devices may not improve tail latency as much as expected, and they can introduce new failure modes such as uneven load distribution. Conversely, better batching and routing can reduce latency without capital spend, though there is a limit: once devices are truly saturated, software elegance cannot create physical compute. The right question is whether our p95 latency is high because we lack capacity or because we are using capacity poorly.\n\nReliability cuts both ways. More machines can provide redundancy and reduce the blast radius of a single failure, which matters if local inference is on the critical path. Yet every additional device adds drivers, thermals, networking, and maintenance to a team that is already small. Improving utilisation can reduce per-device load and make failures less frequent, but pushing utilisation too high can also make the system brittle, because there is less headroom for spikes or degraded performance. We should aim for a deliberate margin rather than maximum occupancy, and we should treat reliability as a measured property, not an assumed benefit of ownership.\n\nOpportunity cost is perhaps the hardest factor to quantify. Engineering hours spent on utilisation are hours not spent on product features, model quality, or customer-facing improvements. Buying hardware can feel like buying time, but it also consumes budget that might fund cloud burst capacity, better tooling, or another hire. For a small team, the scarce resource is often attention, not money. We need to be honest about whether we can execute a utilisation programme without stalling roadmap work. If the answer is no, a modest hardware purchase may be the pragmatic choice, even if it is less elegant.\n\nReversibility favours the software-first approach. Code changes, scheduling policies, and routing rules can be rolled back or adjusted as we learn. Hardware is far less reversible: resale value is uncertain, depreciation is immediate, and operational habits form around whatever we buy. That said, utilisation work is not free of sunk cost, because engineering time cannot be recovered. The sensible compromise is to make the software work time-boxed and measurable, with a clear decision gate. We should instrument current throughput, queue depth, device utilisation, and tail latency; implement the highest-leverage improvements, such as dynamic batching, request routing, and cache reuse; then run for four weeks and compare against explicit thresholds. If sustained utilisation remains above roughly eighty percent and p95 latency still misses target, we should buy. If not, we will have learned that our problem was coordination rather than capacity.\n\nThere is genuine uncertainty in all of this. Demand may grow faster than our models predict, model sizes may change, and local hardware economics may shift. We also do not yet know how much headroom our current stack can realistically recover. That uncertainty is exactly why a reversible experiment is preferable to a large commitment. We should not pretend the data is already decisive, but we can make it decisive within a quarter. I propose we begin with a two-week audit, choose no more than three improvements, and set the purchase decision for the end of the following month. If the audit shows we are already near a hard ceiling, we can accelerate the purchase without losing much time. If it shows waste, we will have avoided an expensive mistake. Either way, the team gains clarity, and clarity is the real asset we are buying.", "output_characters": 5123, "output_sha256": "469822ac6357896084fee8f767c416c1883cb9579cdc2a5269326cfb475e2657", "prompt_tokens": 174, "reasoning_characters": 4024, "reasoning_sha256": "9128e9b6a4a6658ad9d1472b36d4ae6f03658e57a6d7c3ca8108493508a0e2e6", "response_tail_seconds": 0.001019, "run": 4, "stream_done_marker": true, "stream_sha256": "7c1ee29876e962814a6fe0068d681ccf1432f8f5f4d7510c2391154e36611f7b", "time_to_first_visible_seconds": 28.41192, "time_to_last_output_seconds": 61.314618, "ttft_seconds": 0.731653, "wall_seconds": 61.315637 }, { "cached_prompt_tokens": 0, "completion_tokens": 3347, "completion_validation": { "valid": true }, "decode_seconds": 93.065483, "decode_tokens_per_second": 35.953, "finish_reason": "stop", "measured_sse_events": 1155, "output": "MEMORANDUM\n\nTo: Engineering Team\nFrom: Platform Engineering\nDate: Today\nSubject: Buy more local AI hardware or improve utilization first\n\nWe should not buy additional local AI hardware yet. Instead, we should spend a bounded period improving utilization of the machines we already own, while defining explicit conditions that would justify a purchase. This recommendation is not a claim that our current capacity is sufficient; it is a claim that we do not yet know whether capacity or efficiency is the binding constraint. Buying first would commit budget and attention before we have measured the problem, whereas utilization work can reveal the true bottleneck while preserving our options.\n\nCost is the most immediate reason to pause. New hardware carries a purchase price, but also power, cooling, space, maintenance, and replacement costs that are easy to underestimate in a small team. Those costs are fixed even when demand is uneven, and they compete directly with hiring, product work, and tooling. Improving utilization costs engineering time rather than capital, and that time is real, but it does not create a depreciating asset or a recurring facilities obligation. If our machines are idle, poorly scheduled, or running workloads that do not need their full capability, then buying more would simply duplicate waste. We should first determine whether we are paying for capacity we already have.\n\nLatency is the strongest argument for expansion, particularly if users experience slow responses during peaks. Yet latency is not governed by raw compute alone. Queueing discipline, request batching, caching of repeated prompts, model selection, and separating interactive traffic from background jobs can all reduce tail latency without adding silicon. In many small deployments, the bottleneck is not average throughput but contention: a long batch job blocks a short user request, or every request pays for a model larger than necessary. Before concluding that we need more hardware, we should profile where time is spent. If devices are saturated during user-facing traffic after scheduling improvements, capacity is genuinely constrained. If load is uneven or overhead is avoidable, utilization work will likely outperform a purchase.\n\nReliability cuts both ways. More machines can improve availability if used for redundancy, but they also multiply components, firmware versions, thermal edge cases, and operational failure modes. A small team can be overwhelmed by complexity before it is overwhelmed by compute. Improving utilization can reduce contention and prevent overload, but it can also reduce margin if we push existing hardware too hard. We should therefore treat utilization gains as a way to create headroom, not as permission to run everything at maximum. Changes that improve isolation, graceful degradation, and observability should take priority over changes that merely squeeze more work through the same path.\n\nThe deeper trade-off is opportunity cost and reversibility. Engineering hours spent on profiling and scheduling are hours not spent on features, customers, or experiments, and that cost deserves honest accounting. However, hardware purchases also consume opportunity: budget committed to machines cannot fund other bets. Utilization improvements are largely reversible, and a better batching strategy or cache continues to pay off as demand grows. A machine bought for today's peak may sit underused next quarter. If focused work does not move latency or cost per request meaningfully, that is evidence that the bottleneck is capacity rather than efficiency. Until then, cloud bursting can cover exceptional peaks without a permanent commitment.\n\nWe should acknowledge real uncertainty. Demand, model sizes, and product direction may shift, and the balance between capacity and efficiency can change with them. The recommendation is to improve utilization first within a time box, measure against clear targets, and define purchase triggers in advance. If interactive p95 latency remains above target after profiling and scheduling improvements, or if devices are saturated during normal business hours for a sustained period, then we should buy. Until those conditions are met, we should avoid an irreversible capital commitment and build the measurement discipline that will make any future purchase better informed.", "output_characters": 4365, "output_sha256": "6798468d99e00f578e20a07e3ab41c51bca5411758c76988569e94ed8e7a75be", "prompt_tokens": 171, "reasoning_characters": 13043, "reasoning_sha256": "4d061ee32d1c4cd8b9ae84c006b24049d51550b49cfa5a540a159b9d6245e86e", "response_tail_seconds": 0.001627, "run": 5, "stream_done_marker": true, "stream_sha256": "baaea033c4bef222f1f3a9b58158951261578aba4cf47f497feccd5d30a59b23", "time_to_first_visible_seconds": 81.020072, "time_to_last_output_seconds": 93.749323, "ttft_seconds": 0.68384, "wall_seconds": 93.75095 } ], "time_to_last_output_seconds": { "maximum": 107.042, "median": 80.226, "minimum": 54.616, "p90": 107.042 }, "ttft_seconds": { "maximum": 0.732, "median": 0.684, "minimum": 0.671, "p90": 0.732 }, "wall_seconds": { "maximum": 107.045, "median": 80.229, "minimum": 54.618, "p90": 107.045 } }, "structured": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 60.058, "median": 59.419, "minimum": 58.738, "p90": 60.058 }, "pooled_decode_tokens_per_second": 59.448, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 646, "completion_validation": { "valid": true }, "decode_seconds": 10.855048, "decode_tokens_per_second": 59.419, "finish_reason": "stop", "measured_sse_events": 112, "output": "[\n{\"index\":1,\"square\":1},\n{\"index\":2,\"square\":4},\n{\"index\":3,\"square\":9},\n{\"index\":4,\"square\":16},\n{\"index\":5,\"square\":25},\n{\"index\":6,\"square\":36},\n{\"index\":7,\"square\":49},\n{\"index\":8,\"square\":64},\n{\"index\":9,\"square\":81},\n{\"index\":10,\"square\":100},\n{\"index\":11,\"square\":121},\n{\"index\":12,\"square\":144},\n{\"index\":13,\"square\":169},\n{\"index\":14,\"square\":196},\n{\"index\":15,\"square\":225},\n{\"index\":16,\"square\":256},\n{\"index\":17,\"square\":289},\n{\"index\":18,\"square\":324},\n{\"index\":19,\"square\":361},\n{\"index\":20,\"square\":400},\n{\"index\":21,\"square\":441},\n{\"index\":22,\"square\":484},\n{\"index\":23,\"square\":529},\n{\"index\":24,\"square\":576},\n{\"index\":25,\"square\":625},\n{\"index\":26,\"square\":676},\n{\"index\":27,\"square\":729},\n{\"index\":28,\"square\":784},\n{\"index\":29,\"square\":841},\n{\"index\":30,\"square\":900},\n{\"index\":31,\"square\":961},\n{\"index\":32,\"square\":1024},\n{\"index\":33,\"square\":1089},\n{\"index\":34,\"square\":1156},\n{\"index\":35,\"square\":1225},\n{\"index\":36,\"square\":1296},\n{\"index\":37,\"square\":1369},\n{\"index\":38,\"square\":1444},\n{\"index\":39,\"square\":1521},\n{\"index\":40,\"square\":1600},\n{\"index\":41,\"square\":1681},\n{\"index\":42,\"square\":1764},\n{\"index\":43,\"square\":1849},\n{\"index\":44,\"square\":1936},\n{\"index\":45,\"square\":2025},\n{\"index\":46,\"square\":2116},\n{\"index\":47,\"square\":2209},\n{\"index\":48,\"square\":2304},\n{\"index\":49,\"square\":2401},\n{\"index\":50,\"square\":2500}\n]", "output_characters": 1350, "output_sha256": "d1dc8017fd8686282bd91403be19f53fe96d28a3a57bf740e0ae7cdff58a1b64", "prompt_tokens": 160, "reasoning_characters": 147, "reasoning_sha256": "b5871a6c3d83baf5a343ba8a2893d8401a1ceddd3f04480b01561a130488f1be", "response_tail_seconds": 0.000647, "run": 1, "stream_done_marker": true, "stream_sha256": "334908a01a0d191912687c3c72d687e4543582d3d400eed1e5960f996a98087d", "time_to_first_visible_seconds": 1.608126, "time_to_last_output_seconds": 11.513768, "ttft_seconds": 0.65872, "wall_seconds": 11.514414 }, { "cached_prompt_tokens": 0, "completion_tokens": 646, "completion_validation": { "valid": true }, "decode_seconds": 10.900681, "decode_tokens_per_second": 59.171, "finish_reason": "stop", "measured_sse_events": 112, "output": "[\n{\"index\":1,\"square\":1},\n{\"index\":2,\"square\":4},\n{\"index\":3,\"square\":9},\n{\"index\":4,\"square\":16},\n{\"index\":5,\"square\":25},\n{\"index\":6,\"square\":36},\n{\"index\":7,\"square\":49},\n{\"index\":8,\"square\":64},\n{\"index\":9,\"square\":81},\n{\"index\":10,\"square\":100},\n{\"index\":11,\"square\":121},\n{\"index\":12,\"square\":144},\n{\"index\":13,\"square\":169},\n{\"index\":14,\"square\":196},\n{\"index\":15,\"square\":225},\n{\"index\":16,\"square\":256},\n{\"index\":17,\"square\":289},\n{\"index\":18,\"square\":324},\n{\"index\":19,\"square\":361},\n{\"index\":20,\"square\":400},\n{\"index\":21,\"square\":441},\n{\"index\":22,\"square\":484},\n{\"index\":23,\"square\":529},\n{\"index\":24,\"square\":576},\n{\"index\":25,\"square\":625},\n{\"index\":26,\"square\":676},\n{\"index\":27,\"square\":729},\n{\"index\":28,\"square\":784},\n{\"index\":29,\"square\":841},\n{\"index\":30,\"square\":900},\n{\"index\":31,\"square\":961},\n{\"index\":32,\"square\":1024},\n{\"index\":33,\"square\":1089},\n{\"index\":34,\"square\":1156},\n{\"index\":35,\"square\":1225},\n{\"index\":36,\"square\":1296},\n{\"index\":37,\"square\":1369},\n{\"index\":38,\"square\":1444},\n{\"index\":39,\"square\":1521},\n{\"index\":40,\"square\":1600},\n{\"index\":41,\"square\":1681},\n{\"index\":42,\"square\":1764},\n{\"index\":43,\"square\":1849},\n{\"index\":44,\"square\":1936},\n{\"index\":45,\"square\":2025},\n{\"index\":46,\"square\":2116},\n{\"index\":47,\"square\":2209},\n{\"index\":48,\"square\":2304},\n{\"index\":49,\"square\":2401},\n{\"index\":50,\"square\":2500}\n]", "output_characters": 1350, "output_sha256": "d1dc8017fd8686282bd91403be19f53fe96d28a3a57bf740e0ae7cdff58a1b64", "prompt_tokens": 158, "reasoning_characters": 144, "reasoning_sha256": "ca3c0ab8bc0b4708046a1be90aa98e28a4bdac25a4c77fa94bdb889d01fc0e40", "response_tail_seconds": 0.000728, "run": 2, "stream_done_marker": true, "stream_sha256": "485cda2861d8793cd517a3e7c752bfacb2a6b4ccfe93b5aac628b067153b6b29", "time_to_first_visible_seconds": 1.64326, "time_to_last_output_seconds": 11.568985, "ttft_seconds": 0.668304, "wall_seconds": 11.569713 }, { "cached_prompt_tokens": 0, "completion_tokens": 648, "completion_validation": { "valid": true }, "decode_seconds": 10.772965, "decode_tokens_per_second": 60.058, "finish_reason": "stop", "measured_sse_events": 112, "output": "[\n{\"index\":1,\"square\":1},\n{\"index\":2,\"square\":4},\n{\"index\":3,\"square\":9},\n{\"index\":4,\"square\":16},\n{\"index\":5,\"square\":25},\n{\"index\":6,\"square\":36},\n{\"index\":7,\"square\":49},\n{\"index\":8,\"square\":64},\n{\"index\":9,\"square\":81},\n{\"index\":10,\"square\":100},\n{\"index\":11,\"square\":121},\n{\"index\":12,\"square\":144},\n{\"index\":13,\"square\":169},\n{\"index\":14,\"square\":196},\n{\"index\":15,\"square\":225},\n{\"index\":16,\"square\":256},\n{\"index\":17,\"square\":289},\n{\"index\":18,\"square\":324},\n{\"index\":19,\"square\":361},\n{\"index\":20,\"square\":400},\n{\"index\":21,\"square\":441},\n{\"index\":22,\"square\":484},\n{\"index\":23,\"square\":529},\n{\"index\":24,\"square\":576},\n{\"index\":25,\"square\":625},\n{\"index\":26,\"square\":676},\n{\"index\":27,\"square\":729},\n{\"index\":28,\"square\":784},\n{\"index\":29,\"square\":841},\n{\"index\":30,\"square\":900},\n{\"index\":31,\"square\":961},\n{\"index\":32,\"square\":1024},\n{\"index\":33,\"square\":1089},\n{\"index\":34,\"square\":1156},\n{\"index\":35,\"square\":1225},\n{\"index\":36,\"square\":1296},\n{\"index\":37,\"square\":1369},\n{\"index\":38,\"square\":1444},\n{\"index\":39,\"square\":1521},\n{\"index\":40,\"square\":1600},\n{\"index\":41,\"square\":1681},\n{\"index\":42,\"square\":1764},\n{\"index\":43,\"square\":1849},\n{\"index\":44,\"square\":1936},\n{\"index\":45,\"square\":2025},\n{\"index\":46,\"square\":2116},\n{\"index\":47,\"square\":2209},\n{\"index\":48,\"square\":2304},\n{\"index\":49,\"square\":2401},\n{\"index\":50,\"square\":2500}\n]", "output_characters": 1350, "output_sha256": "d1dc8017fd8686282bd91403be19f53fe96d28a3a57bf740e0ae7cdff58a1b64", "prompt_tokens": 156, "reasoning_characters": 158, "reasoning_sha256": "88c46fe18e051424756fd4d85054eec88567a5e9b8bd4d08a3594bbda1d97880", "response_tail_seconds": 0.000166, "run": 3, "stream_done_marker": true, "stream_sha256": "6517435cca842e277bcf2945b1e41157139afbe736e333e9c61d26d72d2b387d", "time_to_first_visible_seconds": 1.59774, "time_to_last_output_seconds": 11.470755, "ttft_seconds": 0.697791, "wall_seconds": 11.470921 }, { "cached_prompt_tokens": 0, "completion_tokens": 596, "completion_validation": { "valid": true }, "decode_seconds": 10.129773, "decode_tokens_per_second": 58.738, "finish_reason": "stop", "measured_sse_events": 105, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 161, "reasoning_characters": 144, "reasoning_sha256": "ca3c0ab8bc0b4708046a1be90aa98e28a4bdac25a4c77fa94bdb889d01fc0e40", "response_tail_seconds": 0.000585, "run": 4, "stream_done_marker": true, "stream_sha256": "eabd7c17542b4affdeefcdb56b24b27e01b85f48a5c789104eafdf3c24ba1f42", "time_to_first_visible_seconds": 1.645035, "time_to_last_output_seconds": 10.831119, "ttft_seconds": 0.701346, "wall_seconds": 10.831703 }, { "cached_prompt_tokens": 0, "completion_tokens": 596, "completion_validation": { "valid": true }, "decode_seconds": 9.941914, "decode_tokens_per_second": 59.848, "finish_reason": "stop", "measured_sse_events": 103, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 159, "reasoning_characters": 147, "reasoning_sha256": "b5871a6c3d83baf5a343ba8a2893d8401a1ceddd3f04480b01561a130488f1be", "response_tail_seconds": 0.000364, "run": 5, "stream_done_marker": true, "stream_sha256": "9bf95b9dd14d3706113ecca00a8e12a572f1905c7b8b3d485c3f9ff2a0471839", "time_to_first_visible_seconds": 1.610324, "time_to_last_output_seconds": 10.594495, "ttft_seconds": 0.652581, "wall_seconds": 10.594859 } ], "time_to_last_output_seconds": { "maximum": 11.569, "median": 11.471, "minimum": 10.594, "p90": 11.569 }, "ttft_seconds": { "maximum": 0.701, "median": 0.668, "minimum": 0.653, "p90": 0.701 }, "wall_seconds": { "maximum": 11.57, "median": 11.471, "minimum": 10.595, "p90": 11.57 } } }, "prefill": { "32768": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 1443.536, "median": 1440.82, "minimum": 1437.891, "p90": 1443.536 }, "runs": [ { "cache_salt_sha256": "0a3feb0f94867db937c4a6a6081686e2787f78a346107b4ec115b701a6b208fe", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.160265, "decode_tokens_per_second": 43.678, "effective_prefill_tokens_per_second": 1437.891, "finish_reason": "length", "measured_sse_events": 3, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.00044, "run": 1, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 22.788938, "time_to_last_output_seconds": 22.949203, "ttft_seconds": 22.788938, "wall_seconds": 22.949643 }, { "cache_salt_sha256": "d5dd996400b600280ace97dd7203ff644aff9c90da6c8ead57d465f140d060fe", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.164715, "decode_tokens_per_second": 42.498, "effective_prefill_tokens_per_second": 1440.82, "finish_reason": "length", "measured_sse_events": 3, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000354, "run": 2, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 22.742599, "time_to_last_output_seconds": 22.907314, "ttft_seconds": 22.742599, "wall_seconds": 22.907668 }, { "cache_salt_sha256": "6cbb4985028fbd39434149da12d72e097e6738c0f46e76b13bb8d472e44a78db", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.16073, "decode_tokens_per_second": 43.551, "effective_prefill_tokens_per_second": 1443.536, "finish_reason": "length", "measured_sse_events": 3, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000124, "run": 3, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 22.699821, "time_to_last_output_seconds": 22.860551, "ttft_seconds": 22.699821, "wall_seconds": 22.860675 } ], "ttft_seconds": { "maximum": 22.789, "median": 22.743, "minimum": 22.7, "p90": 22.789 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 9623.591, "median": 9550.61, "minimum": 9523.068, "p90": 9623.591 }, "runs": [ { "cache_salt_sha256": "0a3feb0f94867db937c4a6a6081686e2787f78a346107b4ec115b701a6b208fe", "cached_prompt_tokens": 28672, "completion_tokens": 8, "decode_seconds": 0.160522, "decode_tokens_per_second": 43.608, "effective_prefill_tokens_per_second": 9550.61, "finish_reason": "length", "measured_sse_events": 3, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 4.3e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 3.430985, "time_to_last_output_seconds": 3.591507, "ttft_seconds": 3.430985, "wall_seconds": 3.59155 }, { "cache_salt_sha256": "d5dd996400b600280ace97dd7203ff644aff9c90da6c8ead57d465f140d060fe", "cached_prompt_tokens": 28672, "completion_tokens": 8, "decode_seconds": 0.157584, "decode_tokens_per_second": 44.421, "effective_prefill_tokens_per_second": 9623.591, "finish_reason": "length", "measured_sse_events": 3, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000125, "run": 2, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 3.404966, "time_to_last_output_seconds": 3.56255, "ttft_seconds": 3.404966, "wall_seconds": 3.562675 }, { "cache_salt_sha256": "6cbb4985028fbd39434149da12d72e097e6738c0f46e76b13bb8d472e44a78db", "cached_prompt_tokens": 28672, "completion_tokens": 8, "decode_seconds": 0.168681, "decode_tokens_per_second": 41.498, "effective_prefill_tokens_per_second": 9523.068, "finish_reason": "length", "measured_sse_events": 3, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000208, "run": 3, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 3.440908, "time_to_last_output_seconds": 3.609589, "ttft_seconds": 3.440908, "wall_seconds": 3.609797 } ], "ttft_seconds": { "maximum": 3.441, "median": 3.431, "minimum": 3.405, "p90": 3.441 } } }, "65536": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 1366.523, "median": 1365.509, "minimum": 1363.633, "p90": 1366.523 }, "runs": [ { "cache_salt_sha256": "4e3cf15ca988bde7a6002e27d9627513ae9ef453ef2fde8791f224df5082d99d", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.16183, "decode_tokens_per_second": 43.255, "effective_prefill_tokens_per_second": 1365.509, "finish_reason": "length", "measured_sse_events": 3, "output": " behaviour, evidence, constraints, and the", "output_characters": 42, "output_sha256": "128002ef37a31be1bbdb67225d75ef3c0c801e9fa83861d8e04741bdff280850", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000231, "run": 1, "stream_done_marker": true, "stream_sha256": "128002ef37a31be1bbdb67225d75ef3c0c801e9fa83861d8e04741bdff280850", "time_to_first_visible_seconds": 47.993832, "time_to_last_output_seconds": 48.155662, "ttft_seconds": 47.993832, "wall_seconds": 48.155893 }, { "cache_salt_sha256": "f0a3ea1e8517a1ab3d07aad01ee32dc708bbb2b6c2fbb6e337bae8cc856d02fd", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.157569, "decode_tokens_per_second": 44.425, "effective_prefill_tokens_per_second": 1363.633, "finish_reason": "length", "measured_sse_events": 3, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000987, "run": 2, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 48.059851, "time_to_last_output_seconds": 48.21742, "ttft_seconds": 48.059851, "wall_seconds": 48.218407 }, { "cache_salt_sha256": "430a4c76c6054c3e4af53f494d3626cdf5463d7727e1b6953f30caafe2ae406c", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.210483, "decode_tokens_per_second": 33.257, "effective_prefill_tokens_per_second": 1366.523, "finish_reason": "length", "measured_sse_events": 4, "output": " and the next action. The repository", "output_characters": 36, "output_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000715, "run": 3, "stream_done_marker": true, "stream_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "time_to_first_visible_seconds": 47.958207, "time_to_last_output_seconds": 48.168691, "ttft_seconds": 47.958207, "wall_seconds": 48.169406 } ], "ttft_seconds": { "maximum": 48.06, "median": 47.994, "minimum": 47.958, "p90": 48.06 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 17532.788, "median": 17443.895, "minimum": 17435.208, "p90": 17532.788 }, "runs": [ { "cache_salt_sha256": "4e3cf15ca988bde7a6002e27d9627513ae9ef453ef2fde8791f224df5082d99d", "cached_prompt_tokens": 61440, "completion_tokens": 8, "decode_seconds": 0.158571, "decode_tokens_per_second": 44.144, "effective_prefill_tokens_per_second": 17443.895, "finish_reason": "length", "measured_sse_events": 3, "output": " behaviour, evidence, constraints, and the", "output_characters": 42, "output_sha256": "128002ef37a31be1bbdb67225d75ef3c0c801e9fa83861d8e04741bdff280850", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000832, "run": 1, "stream_done_marker": true, "stream_sha256": "128002ef37a31be1bbdb67225d75ef3c0c801e9fa83861d8e04741bdff280850", "time_to_first_visible_seconds": 3.756959, "time_to_last_output_seconds": 3.915529, "ttft_seconds": 3.756959, "wall_seconds": 3.916361 }, { "cache_salt_sha256": "f0a3ea1e8517a1ab3d07aad01ee32dc708bbb2b6c2fbb6e337bae8cc856d02fd", "cached_prompt_tokens": 61440, "completion_tokens": 8, "decode_seconds": 0.158122, "decode_tokens_per_second": 44.27, "effective_prefill_tokens_per_second": 17435.208, "finish_reason": "length", "measured_sse_events": 3, "output": " evidence, constraints, and the next action", "output_characters": 43, "output_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000442, "run": 2, "stream_done_marker": true, "stream_sha256": "8e3f2e62aa1aa8b8a9338c5a37dadb78449ad33a44308ac04b7508596128b638", "time_to_first_visible_seconds": 3.758831, "time_to_last_output_seconds": 3.916953, "ttft_seconds": 3.758831, "wall_seconds": 3.917395 }, { "cache_salt_sha256": "430a4c76c6054c3e4af53f494d3626cdf5463d7727e1b6953f30caafe2ae406c", "cached_prompt_tokens": 61440, "completion_tokens": 8, "decode_seconds": 0.210488, "decode_tokens_per_second": 33.256, "effective_prefill_tokens_per_second": 17532.788, "finish_reason": "length", "measured_sse_events": 4, "output": " and the next action. The repository", "output_characters": 36, "output_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000711, "run": 3, "stream_done_marker": true, "stream_sha256": "57be7c5160a81419095c8a5e6cb16166b7f1930a0c7450fdaefb448bcb8925c4", "time_to_first_visible_seconds": 3.737911, "time_to_last_output_seconds": 3.948399, "ttft_seconds": 3.737911, "wall_seconds": 3.94911 } ], "ttft_seconds": { "maximum": 3.759, "median": 3.757, "minimum": 3.738, "p90": 3.759 } } }, "8192": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 1479.975, "median": 1478.009, "minimum": 1477.701, "p90": 1479.975 }, "runs": [ { "cache_salt_sha256": "21535366e4ea106e8fa8fe13675fb56fe5b6aec479425ed88cfeab43e8ead5a7", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.156789, "decode_tokens_per_second": 44.646, "effective_prefill_tokens_per_second": 1478.009, "finish_reason": "length", "measured_sse_events": 3, "output": " tests, documentation, and release notes.", "output_characters": 41, "output_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 4.3e-05, "run": 1, "stream_done_marker": true, "stream_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "time_to_first_visible_seconds": 5.54259, "time_to_last_output_seconds": 5.699379, "ttft_seconds": 5.54259, "wall_seconds": 5.699422 }, { "cache_salt_sha256": "b4c2cb828c49f554681ffd9224a763a5b842e92908ac183d0cce582c2236413f", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.154268, "decode_tokens_per_second": 45.375, "effective_prefill_tokens_per_second": 1479.975, "finish_reason": "length", "measured_sse_events": 3, "output": ", documentation, and release notes. Each", "output_characters": 40, "output_sha256": "e8f6d2eb78c1e344829ed1b2e941d0ca14d7c42bac0c5cb749cd09f9d2ce6814", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 4.7e-05, "run": 2, "stream_done_marker": true, "stream_sha256": "e8f6d2eb78c1e344829ed1b2e941d0ca14d7c42bac0c5cb749cd09f9d2ce6814", "time_to_first_visible_seconds": 5.535229, "time_to_last_output_seconds": 5.689498, "ttft_seconds": 5.535229, "wall_seconds": 5.689545 }, { "cache_salt_sha256": "11b3733535f0c2f9334aaab62bad913da5de098c38148684ee7d16296b2d65a9", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.157497, "decode_tokens_per_second": 44.445, "effective_prefill_tokens_per_second": 1477.701, "finish_reason": "length", "measured_sse_events": 3, "output": ", tests, documentation, and release notes", "output_characters": 41, "output_sha256": "41067174b9993d03b64fee1322cfb0c1ae84ccd92e980017d0765315051c4900", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000258, "run": 3, "stream_done_marker": true, "stream_sha256": "41067174b9993d03b64fee1322cfb0c1ae84ccd92e980017d0765315051c4900", "time_to_first_visible_seconds": 5.543746, "time_to_last_output_seconds": 5.701243, "ttft_seconds": 5.543746, "wall_seconds": 5.701501 } ], "ttft_seconds": { "maximum": 5.544, "median": 5.543, "minimum": 5.535, "p90": 5.544 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 2580.116, "median": 2561.755, "minimum": 2539.598, "p90": 2580.116 }, "runs": [ { "cache_salt_sha256": "21535366e4ea106e8fa8fe13675fb56fe5b6aec479425ed88cfeab43e8ead5a7", "cached_prompt_tokens": 4096, "completion_tokens": 8, "decode_seconds": 0.156741, "decode_tokens_per_second": 44.66, "effective_prefill_tokens_per_second": 2561.755, "finish_reason": "length", "measured_sse_events": 3, "output": " tests, documentation, and release notes.", "output_characters": 41, "output_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000482, "run": 1, "stream_done_marker": true, "stream_sha256": "052dc6176cb0a2e719a14647a717dfa4d0121a0bb7402d3dff7dd2217852cd4d", "time_to_first_visible_seconds": 3.197808, "time_to_last_output_seconds": 3.35455, "ttft_seconds": 3.197808, "wall_seconds": 3.355032 }, { "cache_salt_sha256": "b4c2cb828c49f554681ffd9224a763a5b842e92908ac183d0cce582c2236413f", "cached_prompt_tokens": 4096, "completion_tokens": 8, "decode_seconds": 0.155235, "decode_tokens_per_second": 45.093, "effective_prefill_tokens_per_second": 2539.598, "finish_reason": "length", "measured_sse_events": 3, "output": ", documentation, and release notes. Each", "output_characters": 40, "output_sha256": "e8f6d2eb78c1e344829ed1b2e941d0ca14d7c42bac0c5cb749cd09f9d2ce6814", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000289, "run": 2, "stream_done_marker": true, "stream_sha256": "e8f6d2eb78c1e344829ed1b2e941d0ca14d7c42bac0c5cb749cd09f9d2ce6814", "time_to_first_visible_seconds": 3.225707, "time_to_last_output_seconds": 3.380941, "ttft_seconds": 3.225707, "wall_seconds": 3.38123 }, { "cache_salt_sha256": "11b3733535f0c2f9334aaab62bad913da5de098c38148684ee7d16296b2d65a9", "cached_prompt_tokens": 4096, "completion_tokens": 8, "decode_seconds": 0.155853, "decode_tokens_per_second": 44.914, "effective_prefill_tokens_per_second": 2580.116, "finish_reason": "length", "measured_sse_events": 3, "output": ", tests, documentation, and release notes", "output_characters": 41, "output_sha256": "41067174b9993d03b64fee1322cfb0c1ae84ccd92e980017d0765315051c4900", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.00064, "run": 3, "stream_done_marker": true, "stream_sha256": "41067174b9993d03b64fee1322cfb0c1ae84ccd92e980017d0765315051c4900", "time_to_first_visible_seconds": 3.175051, "time_to_last_output_seconds": 3.330904, "ttft_seconds": 3.175051, "wall_seconds": 3.331544 } ], "ttft_seconds": { "maximum": 3.226, "median": 3.198, "minimum": 3.175, "p90": 3.226 } } } }, "protocol": { "prompts_sha256": "0c3ac4015813af6a1b967659bd2289990fde895a5e43ae065b894d1e1f909ecb", "prompts_version": "1.0.0", "repository_dirty": false, "repository_revision": "946394fa53db83a6865f6e3ae56594e1c299fb2a", "repository_source": "git-worktree", "repository_source_sha256": "f5503bd3fe66d96d42992fedb6cf0b47f1e7ba560359d455a6e8fc1c471261d7", "version": "1.3.0" }, "run": { "appliance": { "competing_traffic": "none intended: owner-gateway quiet window (only ALBERT+MINIME keys); client: FrankenNAS over 1 GbE LAN to the lane's rank-0 port (same path for every lane)", "context_limit": 1048576, "drafter": "DFlash2 (Beam-1.0-501B-A23B-DFlash2)", "hardware": "2x NVIDIA DGX Spark (GB10, 128 GB unified memory each), ascent-03 + ascent-04", "kv_cache_dtype": "engine default for this lane", "model": "bertholomus/Beam-1.0-501B-A23B-EXL3-3bpw-2xGB10", "model_revision": "local folder Beam-1.0-501B-A23B-EXL3-2spark-k3c36", "negotiated_link_speed": "200 Gb/s per CX7 function", "quantisation": "EXL3 3 bpw weights, BF16 activations", "scheduler": "lane as served in the bake-off, unchanged", "serving_engine": "TensorFold fork github.com/bertholomus/TensorFold branch beam-1.0-tp2 lineage", "serving_image": "see serving_engine", "topology": "TP2, one rank per Spark, RoCE over ConnectX-7" }, "comparison_id": "2026-10-two-sparks-four-models-v1", "finished_at": "2026-10-29T03:18:44.832679+00:00", "label": "bakeoff-beam", "model": "beam-1.0", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.43", "python": "3.11.15", "rigmark_release_check": { "ahead_by": 6, "behind_by": 0, "dirty": false, "prerelease": true, "release_url": "https://github.com/alexellis/rigmark/releases/tag/v0.0.4", "revision": "946394fa53db83a6865f6e3ae56594e1c299fb2a", "status": "ahead", "tag": "v0.0.4" }, "started_at": "2026-10-29T02:53:38.629287+00:00" }, "schema": 1, "settings": { "concurrency": [ 1, 2, 4 ], "concurrency_runs": 3, "concurrency_tokens": 256, "concurrency_workload": "code", "decode_tokens": 8192, "extra_body": {}, "prefill_cache_isolation": "random-salt-per-pair", "prefill_depths": [ 8192, 32768, 65536 ], "prefill_runs": 3, "runs": 5, "seed": 20260905, "temperature": 0.0, "top_p": 1.0 } }