{ "concurrency": { "1": { "aggregate_end_to_end_tokens_per_second": { "maximum": 78.964, "median": 74.181, "minimum": 73.88, "p90": 78.964 }, "per_stream_decode_tokens_per_second": { "maximum": 88.698, "median": 82.646, "minimum": 82.325, "p90": 88.698 }, "per_stream_ttft_seconds": { "maximum": 0.367, "median": 0.367, "minimum": 0.365, "p90": 0.367 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 73.88, "completion_tokens": 256, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.097471, "decode_tokens_per_second": 82.325, "finish_reason": "length", "measured_sse_events": 90, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 147, "reasoning_characters": 1137, "reasoning_sha256": "79e4488b5380726765937fb4d9a9bd6250221d3e014f2d81c3ff50d6d026bb29", "response_tail_seconds": 0.000161, "stream_done_marker": true, "stream_sha256": "79e4488b5380726765937fb4d9a9bd6250221d3e014f2d81c3ff50d6d026bb29", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 3.464515, "ttft_seconds": 0.367044, "wall_seconds": 3.464676 } ], "wall_seconds": 3.465099 }, { "aggregate_end_to_end_tokens_per_second": 78.964, "completion_tokens": 256, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 2.874932, "decode_tokens_per_second": 88.698, "finish_reason": "length", "measured_sse_events": 86, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 145, "reasoning_characters": 1146, "reasoning_sha256": "be65a20a6ccb646ec20924f4b9c464b61f9764524e931d631feab9f984ff90e7", "response_tail_seconds": 0.000139, "stream_done_marker": true, "stream_sha256": "be65a20a6ccb646ec20924f4b9c464b61f9764524e931d631feab9f984ff90e7", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 3.241479, "ttft_seconds": 0.366547, "wall_seconds": 3.241618 } ], "wall_seconds": 3.24197 }, { "aggregate_end_to_end_tokens_per_second": 74.181, "completion_tokens": 256, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.085464, "decode_tokens_per_second": 82.646, "finish_reason": "length", "measured_sse_events": 92, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 150, "reasoning_characters": 1100, "reasoning_sha256": "4b56e4c09f1055bb2863d8d8a112a041c6191839ab6e52e99cd5c8401c256162", "response_tail_seconds": 0.000174, "stream_done_marker": true, "stream_sha256": "4b56e4c09f1055bb2863d8d8a112a041c6191839ab6e52e99cd5c8401c256162", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 3.450465, "ttft_seconds": 0.365001, "wall_seconds": 3.450639 } ], "wall_seconds": 3.451016 } ] }, "2": { "aggregate_end_to_end_tokens_per_second": { "maximum": 105.605, "median": 104.7, "minimum": 104.42, "p90": 105.605 }, "per_stream_decode_tokens_per_second": { "maximum": 77.629, "median": 61.663, "minimum": 56.243, "p90": 67.246 }, "per_stream_ttft_seconds": { "maximum": 0.735, "median": 0.551, "minimum": 0.363, "p90": 0.733 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 105.605, "completion_tokens": 512, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.114409, "decode_tokens_per_second": 61.977, "finish_reason": "length", "measured_sse_events": 96, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 152, "reasoning_characters": 1122, "reasoning_sha256": "f836788912772eb1a9e54181a8ce56152a9e6194cba50afff1ff0a05c50b75bb", "response_tail_seconds": 0.000127, "stream_done_marker": true, "stream_sha256": "f836788912772eb1a9e54181a8ce56152a9e6194cba50afff1ff0a05c50b75bb", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 4.847369, "ttft_seconds": 0.73296, "wall_seconds": 4.847496 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.79206, "decode_tokens_per_second": 67.246, "finish_reason": "length", "measured_sse_events": 75, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 147, "reasoning_characters": 1134, "reasoning_sha256": "54d33cbf067fc5e1e2a83012cc55a3b17bf52424692d204892183a83a5cb8301", "response_tail_seconds": 0.000283, "stream_done_marker": true, "stream_sha256": "54d33cbf067fc5e1e2a83012cc55a3b17bf52424692d204892183a83a5cb8301", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 4.154588, "ttft_seconds": 0.362527, "wall_seconds": 4.154871 } ], "wall_seconds": 4.848247 }, { "aggregate_end_to_end_tokens_per_second": 104.42, "completion_tokens": 512, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 3.284846, "decode_tokens_per_second": 77.629, "finish_reason": "length", "measured_sse_events": 77, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 148, "reasoning_characters": 1126, "reasoning_sha256": "07f87dd2b233c012ddfe772ce27cca59a0a0dc44bbe63362d4d8b946bb95f01d", "response_tail_seconds": 0.000293, "stream_done_marker": true, "stream_sha256": "07f87dd2b233c012ddfe772ce27cca59a0a0dc44bbe63362d4d8b946bb95f01d", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 4.019556, "ttft_seconds": 0.73471, "wall_seconds": 4.019849 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.533905, "decode_tokens_per_second": 56.243, "finish_reason": "length", "measured_sse_events": 105, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 149, "reasoning_characters": 1183, "reasoning_sha256": "647bfb4bc38898a8a51258c6b1b6725f0b4ac58e2f9c72ed7452c99b0ef70a9c", "response_tail_seconds": 0.000172, "stream_done_marker": true, "stream_sha256": "647bfb4bc38898a8a51258c6b1b6725f0b4ac58e2f9c72ed7452c99b0ef70a9c", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 4.902627, "ttft_seconds": 0.368722, "wall_seconds": 4.902799 } ], "wall_seconds": 4.903285 }, { "aggregate_end_to_end_tokens_per_second": 104.7, "completion_tokens": 512, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.269424, "decode_tokens_per_second": 59.727, "finish_reason": "length", "measured_sse_events": 89, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 149, "reasoning_characters": 1160, "reasoning_sha256": "2eeea5412bd2eb2068c3fd1987019d3f44526a842c7f377a74be76aa1deed786", "response_tail_seconds": 0.000378, "stream_done_marker": true, "stream_sha256": "2eeea5412bd2eb2068c3fd1987019d3f44526a842c7f377a74be76aa1deed786", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 4.638439, "ttft_seconds": 0.369015, "wall_seconds": 4.638817 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 4.156569, "decode_tokens_per_second": 61.349, "finish_reason": "length", "measured_sse_events": 97, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 148, "reasoning_characters": 1161, "reasoning_sha256": "bdcf6b653c5a93964dfa9463595a93196fb0771b4617dc969b4adea187e061fa", "response_tail_seconds": 0.000141, "stream_done_marker": true, "stream_sha256": "bdcf6b653c5a93964dfa9463595a93196fb0771b4617dc969b4adea187e061fa", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 4.889648, "ttft_seconds": 0.733079, "wall_seconds": 4.889789 } ], "wall_seconds": 4.890176 } ] }, "4": { "aggregate_end_to_end_tokens_per_second": { "maximum": 142.45, "median": 139.738, "minimum": 136.559, "p90": 142.45 }, "per_stream_decode_tokens_per_second": { "maximum": 48.976, "median": 42.59, "minimum": 37.385, "p90": 47.929 }, "per_stream_ttft_seconds": { "maximum": 1.469, "median": 0.915, "minimum": 0.362, "p90": 1.461 }, "rounds": [ { "aggregate_end_to_end_tokens_per_second": 142.45, "completion_tokens": 1024, "round": 1, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.024472, "decode_tokens_per_second": 42.327, "finish_reason": "length", "measured_sse_events": 95, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 147, "reasoning_characters": 1181, "reasoning_sha256": "c9d25730695e0691f7fd3d52eb7a540c280e8ed3bc846b5d97c06aad623961a5", "response_tail_seconds": 0.000285, "stream_done_marker": true, "stream_sha256": "c9d25730695e0691f7fd3d52eb7a540c280e8ed3bc846b5d97c06aad623961a5", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.122332, "ttft_seconds": 1.09786, "wall_seconds": 7.122617 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.206617, "decode_tokens_per_second": 48.976, "finish_reason": "length", "measured_sse_events": 85, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 146, "reasoning_characters": 1153, "reasoning_sha256": "4a37649753437a4cdc9e04c41b19c7ff695d25a741cf356322c8de2a98dfb48c", "response_tail_seconds": 0.000231, "stream_done_marker": true, "stream_sha256": "4a37649753437a4cdc9e04c41b19c7ff695d25a741cf356322c8de2a98dfb48c", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.668005, "ttft_seconds": 1.461388, "wall_seconds": 6.668236 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.934218, "decode_tokens_per_second": 42.971, "finish_reason": "length", "measured_sse_events": 85, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 150, "reasoning_characters": 1147, "reasoning_sha256": "9ebc570f3b217a3f939dbdf4573f30632fe60dc8278e1bb27b74847749978e51", "response_tail_seconds": 0.000275, "stream_done_marker": true, "stream_sha256": "9ebc570f3b217a3f939dbdf4573f30632fe60dc8278e1bb27b74847749978e51", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.668054, "ttft_seconds": 0.733836, "wall_seconds": 6.668329 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.820864, "decode_tokens_per_second": 37.385, "finish_reason": "length", "measured_sse_events": 97, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 148, "reasoning_characters": 1107, "reasoning_sha256": "a7f317366f168832b77988943f0515c017923d826df85fffafa2b8dec7e2ef1e", "response_tail_seconds": 0.000169, "stream_done_marker": true, "stream_sha256": "a7f317366f168832b77988943f0515c017923d826df85fffafa2b8dec7e2ef1e", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.187568, "ttft_seconds": 0.366704, "wall_seconds": 7.187737 } ], "wall_seconds": 7.18849 }, { "aggregate_end_to_end_tokens_per_second": 136.559, "completion_tokens": 1024, "round": 2, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.592438, "decode_tokens_per_second": 45.597, "finish_reason": "length", "measured_sse_events": 91, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 147, "reasoning_characters": 1157, "reasoning_sha256": "5c9bcf5fa9cb5fba1cb0b703130d47d50c8eba03e8d60e7d29922ea9fa68cd41", "response_tail_seconds": 0.000175, "stream_done_marker": true, "stream_sha256": "5c9bcf5fa9cb5fba1cb0b703130d47d50c8eba03e8d60e7d29922ea9fa68cd41", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.061633, "ttft_seconds": 1.469195, "wall_seconds": 7.061808 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.950581, "decode_tokens_per_second": 42.853, "finish_reason": "length", "measured_sse_events": 84, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 150, "reasoning_characters": 1147, "reasoning_sha256": "e0a6055a8357fa714e73ce8d467de3b1679336066b1126e5e3f248465ad46899", "response_tail_seconds": 0.000207, "stream_done_marker": true, "stream_sha256": "e0a6055a8357fa714e73ce8d467de3b1679336066b1126e5e3f248465ad46899", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.689952, "ttft_seconds": 0.73937, "wall_seconds": 6.690158 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.395767, "decode_tokens_per_second": 39.87, "finish_reason": "length", "measured_sse_events": 106, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 149, "reasoning_characters": 1223, "reasoning_sha256": "b3e78c6476eef1d8615447eda27712eb51dad0f56983856d452d6248a2f49f18", "response_tail_seconds": 0.00015, "stream_done_marker": true, "stream_sha256": "b3e78c6476eef1d8615447eda27712eb51dad0f56983856d452d6248a2f49f18", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.497681, "ttft_seconds": 1.101914, "wall_seconds": 7.497831 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.657712, "decode_tokens_per_second": 38.301, "finish_reason": "length", "measured_sse_events": 90, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 150, "reasoning_characters": 1140, "reasoning_sha256": "55c03d982ab2aa3e08c388299dfb8fe6a24b5953a959fdf78bb42ec8f5d9dce9", "response_tail_seconds": 0.000188, "stream_done_marker": true, "stream_sha256": "55c03d982ab2aa3e08c388299dfb8fe6a24b5953a959fdf78bb42ec8f5d9dce9", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.024372, "ttft_seconds": 0.36666, "wall_seconds": 7.024559 } ], "wall_seconds": 7.498586 }, { "aggregate_end_to_end_tokens_per_second": 139.738, "completion_tokens": 1024, "round": 3, "streams": [ { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.320333, "decode_tokens_per_second": 47.929, "finish_reason": "length", "measured_sse_events": 84, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 149, "reasoning_characters": 1124, "reasoning_sha256": "eea47636c660b911e96a4bcbed3c87ac3c9106a106c37fc4d52d555bd935a714", "response_tail_seconds": 0.000175, "stream_done_marker": true, "stream_sha256": "eea47636c660b911e96a4bcbed3c87ac3c9106a106c37fc4d52d555bd935a714", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.775405, "ttft_seconds": 1.455072, "wall_seconds": 6.77558 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.602016, "decode_tokens_per_second": 38.625, "finish_reason": "length", "measured_sse_events": 100, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 152, "reasoning_characters": 1163, "reasoning_sha256": "71e136fbf2d5750dec36de32ab6c75b8536552c59619dbdd4563a9b45e88dc28", "response_tail_seconds": 0.000148, "stream_done_marker": true, "stream_sha256": "71e136fbf2d5750dec36de32ab6c75b8536552c59619dbdd4563a9b45e88dc28", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 7.326862, "ttft_seconds": 0.724846, "wall_seconds": 7.327011 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 5.869754, "decode_tokens_per_second": 43.443, "finish_reason": "length", "measured_sse_events": 88, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 146, "reasoning_characters": 1143, "reasoning_sha256": "a9463d150daf34ef4c8d76f16bd350ac526f480c2a1a0c07e39be16c09307a8d", "response_tail_seconds": 0.000301, "stream_done_marker": true, "stream_sha256": "a9463d150daf34ef4c8d76f16bd350ac526f480c2a1a0c07e39be16c09307a8d", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.960835, "ttft_seconds": 1.091082, "wall_seconds": 6.961136 }, { "cached_prompt_tokens": 0, "completion_tokens": 256, "decode_seconds": 6.412964, "decode_tokens_per_second": 39.763, "finish_reason": "length", "measured_sse_events": 84, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 150, "reasoning_characters": 1160, "reasoning_sha256": "db57c6e1f8cefecf9d4d345ba26138c38acc8ba327d2a9a9cecee4400b32aa6c", "response_tail_seconds": 0.000209, "stream_done_marker": true, "stream_sha256": "db57c6e1f8cefecf9d4d345ba26138c38acc8ba327d2a9a9cecee4400b32aa6c", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 6.775444, "ttft_seconds": 0.36248, "wall_seconds": 6.775653 } ], "wall_seconds": 7.327977 } ] } }, "decode": { "code": { "completion_gate": { "passed": 2, "total": 5 }, "decode_tokens_per_second": { "maximum": 111.674, "median": 99.502, "minimum": 98.268, "p90": 111.674 }, "pooled_decode_tokens_per_second": 103.417, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 4880, "completion_validation": { "valid": true }, "decode_seconds": 49.034175, "decode_tokens_per_second": 99.502, "finish_reason": "stop", "measured_sse_events": 1358, "output": "```go\n// limiter.go\npackage ratelimit\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype Clock interface {\n\tNow() time.Time\n}\n\ntype Limiter struct {\n\tmu sync.Mutex\n\tcapacity float64\n\trate float64\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\nfunc NewLimiter(capacity int, rate float64, clock Clock) (*Limiter, error) {\n\tif capacity <= 0 {\n\t\treturn nil, errors.New(\"ratelimit: capacity must be positive\")\n\t}\n\tif rate <= 0 || math.IsNaN(rate) || math.IsInf(rate, 0) {\n\t\treturn nil, errors.New(\"ratelimit: rate must be positive and finite\")\n\t}\n\tif clock == nil {\n\t\treturn nil, errors.New(\"ratelimit: clock must not be nil\")\n\t}\n\treturn &Limiter{\n\t\tcapacity: float64(capacity),\n\t\trate: rate,\n\t\ttokens: float64(capacity),\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn false\n\t}\n\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\tif now.Before(l.last) {\n\t\tl.last = now\n\t} else if elapsed := now.Sub(l.last).Seconds(); elapsed > 0 {\n\t\tl.tokens = math.Min(l.capacity, l.tokens+elapsed*l.rate)\n\t\tl.last = now\n\t}\n\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// limiter_test.go\npackage ratelimit\n\nimport (\n\t\"math\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tnow time.Time\n}\n\nfunc newFakeClock(t time.Time) *fakeClock {\n\treturn &fakeClock{now: t}\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.now\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.now = c.now.Add(d)\n}\n\nfunc TestNewLimiterValidation(t *testing.T) {\n\tclock := newFakeClock(time.Unix(0, 0))\n\ttests := []struct {\n\t\tname string\n\t\tcapacity int\n\t\trate float64\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 1, 1, clock, false},\n\t\t{\"zero capacity\", 0, 1, clock, true},\n\t\t{\"negative capacity\", -1, 1, clock, true},\n\t\t{\"zero rate\", 1, 0, clock, true},\n\t\t{\"negative rate\", 1, -1, clock, true},\n\t\t{\"nan rate\", 1, math.NaN(), clock, true},\n\t\t{\"inf rate\", 1, math.Inf(1), clock, true},\n\t\t{\"nil clock\", 1, 1, nil, true},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := NewLimiter(tt.capacity, tt.rate, tt.clock)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Fatalf(\"err=%v, wantErr=%v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcapacity int\n\t\trate float64\n\t\tsteps []struct {\n\t\t\tadvance time.Duration\n\t\t\tn int\n\t\t\twant bool\n\t\t}\n\t}{\n\t\t{\n\t\t\tname: \"consume and refill\",\n\t\t\tcapacity: 2,\n\t\t\trate: 1,\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twant bool\n\t\t\t}{\n\t\t\t\t{0, 1, true},\n\t\t\t\t{0, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{time.Second, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{time.Second, 1, true},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"capacity cap\",\n\t\t\tcapacity: 2,\n\t\t\trate: 1,\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twant bool\n\t\t\t}{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{10 * time.Second, 1, true},\n\t\t\t\t{0, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"invalid n\",\n\t\t\tcapacity: 1,\n\t\t\trate: 1,\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twant bool\n\t\t\t}{\n\t\t\t\t{0, 0, false},\n\t\t\t\t{0, -1, false},\n\t\t\t\t{0, 1, true},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"clock backwards\",\n\t\t\tcapacity: 1,\n\t\t\trate: 1,\n\t\t\tsteps: []struct {\n\t\t\t\tadvance time.Duration\n\t\t\t\tn int\n\t\t\t\twant bool\n\t\t\t}{\n\t\t\t\t{0, 1, true},\n\t\t\t\t{time.Second, 1, true},\n\t\t\t\t{-2 * time.Second, 1, false},\n\t\t\t\t{time.Second, 1, true},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tclock := newFakeClock(time.Unix(0, 0))\n\t\t\tlim, err := NewLimiter(tt.capacity, tt.rate, clock)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tfor i, step := range tt.steps {\n\t\t\t\tclock.Advance(step.advance)\n\t\t\t\tif got := lim.AllowN(step.n); got != step.want {\n\t\t\t\t\tt.Fatalf(\"step %d: AllowN(%d)=%v, want %v\", i, step.n, got, step.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n```", "output_characters": 4018, "output_sha256": "959bea2a5b01e1c2aa0889ef6b115db83451c93db38084e83c089be9b707741b", "prompt_tokens": 147, "reasoning_characters": 12706, "reasoning_sha256": "d9f873650fe9176752d68d1371e008a1ac1cecc9c9f1b90b06eb40ed937991bd", "response_tail_seconds": 0.034459, "run": 1, "stream_done_marker": true, "stream_sha256": "973f4b0794c9423a0b190fff4987dcfb480005db0a57f51f593f4669a5a55ab1", "time_to_first_visible_seconds": 37.653968, "time_to_last_output_seconds": 49.431168, "ttft_seconds": 0.396993, "wall_seconds": 49.465627 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 76.909975, "decode_tokens_per_second": 106.501, "finish_reason": "length", "measured_sse_events": 2057, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 148, "reasoning_characters": 31256, "reasoning_sha256": "ee671e9623e920f0452129d67269ebedc129a5adbbb3707cbd062db05227e96d", "response_tail_seconds": 0.000845, "run": 2, "stream_done_marker": true, "stream_sha256": "ee671e9623e920f0452129d67269ebedc129a5adbbb3707cbd062db05227e96d", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 77.28086, "ttft_seconds": 0.370885, "wall_seconds": 77.281705 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 73.34753, "decode_tokens_per_second": 111.674, "finish_reason": "length", "measured_sse_events": 1932, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 150, "reasoning_characters": 34509, "reasoning_sha256": "91d02a1e473ac7ab156c25268fcb43d23e00e7bd12565a190e65390355b6645a", "response_tail_seconds": 0.000855, "run": 3, "stream_done_marker": true, "stream_sha256": "91d02a1e473ac7ab156c25268fcb43d23e00e7bd12565a190e65390355b6645a", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 73.718695, "ttft_seconds": 0.371165, "wall_seconds": 73.71955 }, { "cached_prompt_tokens": 0, "completion_tokens": 6036, "completion_validation": { "valid": true }, "decode_seconds": 61.413674, "decode_tokens_per_second": 98.268, "finish_reason": "stop", "measured_sse_events": 1703, "output": "```go\n// tokenbucket.go\npackage tokenbucket\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Clock supplies the current time.\ntype Clock interface {\n\tNow() time.Time\n}\n\n// Limiter is a concurrency-safe token bucket.\ntype Limiter struct {\n\tmu sync.Mutex\n\trate float64\n\tburst float64\n\ttokens float64\n\tlast time.Time\n\tclock Clock\n}\n\n// New creates a Limiter with a full bucket.\nfunc New(rate float64, burst int, clock Clock) (*Limiter, error) {\n\tif clock == nil {\n\t\treturn nil, errors.New(\"tokenbucket: nil clock\")\n\t}\n\tif rate <= 0 || math.IsNaN(rate) || math.IsInf(rate, 0) {\n\t\treturn nil, errors.New(\"tokenbucket: rate must be positive and finite\")\n\t}\n\tif burst <= 0 {\n\t\treturn nil, errors.New(\"tokenbucket: burst must be positive\")\n\t}\n\treturn &Limiter{\n\t\trate: rate,\n\t\tburst: float64(burst),\n\t\ttokens: float64(burst),\n\t\tlast: clock.Now(),\n\t\tclock: clock,\n\t}, nil\n}\n\n// AllowN reports whether n tokens are available and consumes them.\nfunc (l *Limiter) AllowN(n int) bool {\n\tif n <= 0 {\n\t\treturn false\n\t}\n\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\n\tnow := l.clock.Now()\n\tif now.After(l.last) {\n\t\tl.tokens += now.Sub(l.last).Seconds() * l.rate\n\t\tif l.tokens > l.burst {\n\t\t\tl.tokens = l.burst\n\t\t}\n\t\tl.last = now\n\t} else if now.Before(l.last) {\n\t\tl.last = now\n\t}\n\n\tif l.tokens >= float64(n) {\n\t\tl.tokens -= float64(n)\n\t\treturn true\n\t}\n\treturn false\n}\n```\n\n```go\n// tokenbucket_test.go\npackage tokenbucket\n\nimport (\n\t\"math\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype fakeClock struct {\n\tmu sync.Mutex\n\tnow time.Time\n}\n\nfunc newFakeClock(t time.Time) *fakeClock {\n\treturn &fakeClock{now: t}\n}\n\nfunc (c *fakeClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.now\n}\n\nfunc (c *fakeClock) Advance(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.now = c.now.Add(d)\n}\n\nfunc TestNewValidation(t *testing.T) {\n\tfc := newFakeClock(time.Unix(0, 0))\n\ttests := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tclock Clock\n\t\twantErr bool\n\t}{\n\t\t{\"valid\", 1, 1, fc, false},\n\t\t{\"zero rate\", 0, 1, fc, true},\n\t\t{\"negative rate\", -1, 1, fc, true},\n\t\t{\"NaN rate\", math.NaN(), 1, fc, true},\n\t\t{\"Inf rate\", math.Inf(1), 1, fc, true},\n\t\t{\"zero burst\", 1, 0, fc, true},\n\t\t{\"negative burst\", 1, -1, fc, true},\n\t\t{\"nil clock\", 1, 1, nil, true},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tl, err := New(tt.rate, tt.burst, tt.clock)\n\t\t\tif tt.wantErr {\n\t\t\t\tif err == nil || l != nil {\n\t\t\t\t\tt.Fatalf(\"New() = (%v, %v), want error and nil limiter\", l, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err != nil || l == nil {\n\t\t\t\tt.Fatalf(\"New() = (%v, %v), want limiter and nil error\", l, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllowN(t *testing.T) {\n\ttype step struct {\n\t\tadvance time.Duration\n\t\tn int\n\t\twant bool\n\t}\n\ttests := []struct {\n\t\tname string\n\t\trate float64\n\t\tburst int\n\t\tsteps []step\n\t}{\n\t\t{\n\t\t\tname: \"initial burst\",\n\t\t\trate: 1,\n\t\t\tburst: 3,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{0, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"refill\",\n\t\t\trate: 2,\n\t\t\tburst: 5,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 5, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{500 * time.Millisecond, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"partial refill\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{1500 * time.Millisecond, 1, true},\n\t\t\t\t{0, 1, false},\n\t\t\t\t{500 * time.Millisecond, 1, true},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"cap at burst\",\n\t\t\trate: 10,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{time.Second, 2, true},\n\t\t\t\t{0, 1, false},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"non-positive n\",\n\t\t\trate: 1,\n\t\t\tburst: 1,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 0, false},\n\t\t\t\t{0, -1, false},\n\t\t\t\t{0, 1, true},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"clock backwards\",\n\t\t\trate: 1,\n\t\t\tburst: 2,\n\t\t\tsteps: []step{\n\t\t\t\t{0, 2, true},\n\t\t\t\t{-time.Second, 1, false},\n\t\t\t\t{2 * time.Second, 1, true},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfc := newFakeClock(time.Unix(0, 0))\n\t\t\tl, err := New(tt.rate, tt.burst, fc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tfor i, s := range tt.steps {\n\t\t\t\tfc.Advance(s.advance)\n\t\t\t\tif got := l.AllowN(s.n); got != s.want {\n\t\t\t\t\tt.Fatalf(\"step %d: AllowN(%d) = %v, want %v\", i, s.n, got, s.want)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConcurrentAllowN(t *testing.T) {\n\tfc := newFakeClock(time.Unix(0, 0))\n\tl, err := New(1, 100, fc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tvar mu sync.Mutex\n\tallowed := 0\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tif l.AllowN(1) {\n\t\t\t\tmu.Lock()\n\t\t\t\tallowed++\n\t\t\t\tmu.Unlock()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\n\tif allowed != 100 {\n\t\tt.Fatalf(\"allowed = %d, want 100\", allowed)\n\t}\n}\n```", "output_characters": 4628, "output_sha256": "5c2e638ddd440f26ead9d8ba0a9e8c5776d51e207fa590ebb8f0aeebc8f813ad", "prompt_tokens": 149, "reasoning_characters": 15916, "reasoning_sha256": "f951631ca29ddc0f271b2851ea8c4dca3736c19ffd6748f323503cb74cc16998", "response_tail_seconds": 0.000596, "run": 4, "stream_done_marker": true, "stream_sha256": "9619cc3578741ea7b8657fbe04ecdbb8cc3e0181ddbf9f870236cc8de7a5ba1d", "time_to_first_visible_seconds": 47.684043, "time_to_last_output_seconds": 61.784426, "ttft_seconds": 0.370752, "wall_seconds": 61.785022 }, { "cached_prompt_tokens": 0, "completion_tokens": 8192, "completion_validation": { "error": "no visible answer", "valid": false }, "decode_seconds": 82.440665, "decode_tokens_per_second": 99.356, "finish_reason": "length", "measured_sse_events": 2302, "output": "", "output_characters": 0, "output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "prompt_tokens": 149, "reasoning_characters": 33826, "reasoning_sha256": "17da22e755498cbb0beacac5792378a9a0c9ee97fac6860c91227910564852b6", "response_tail_seconds": 0.000773, "run": 5, "stream_done_marker": true, "stream_sha256": "17da22e755498cbb0beacac5792378a9a0c9ee97fac6860c91227910564852b6", "time_to_first_visible_seconds": null, "time_to_last_output_seconds": 82.815504, "ttft_seconds": 0.374839, "wall_seconds": 82.816277 } ], "time_to_last_output_seconds": { "maximum": 82.816, "median": 73.719, "minimum": 49.431, "p90": 82.816 }, "ttft_seconds": { "maximum": 0.397, "median": 0.371, "minimum": 0.371, "p90": 0.397 }, "wall_seconds": { "maximum": 82.816, "median": 73.72, "minimum": 49.466, "p90": 82.816 } }, "prose": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 91.895, "median": 80.824, "minimum": 66.481, "p90": 91.895 }, "pooled_decode_tokens_per_second": 82.591, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 4383, "completion_validation": { "valid": true }, "decode_seconds": 47.684688, "decode_tokens_per_second": 91.895, "finish_reason": "stop", "measured_sse_events": 1413, "output": "**MEMORANDUM**\n\n**To:** Engineering Leadership \n**From:** Infrastructure Engineering \n**Subject:** Local AI capacity: improve utilisation before expanding hardware\n\nThis memo recommends that we improve utilisation of our existing local AI hardware before buying more, and that we make any expansion decision only after a short, measured utilisation programme with explicit decision gates. The choice is not whether hardware is useful; it is whether additional hardware is the cheapest reliable way to meet latency and throughput targets given our current workload and a small team. At present we lack the evidence to justify expansion, and the software and operational work needed to use existing capacity well is both cheaper and more reversible.\n\nCost is the first consideration, but not only the purchase price. Additional local AI hardware carries capital expense, power and cooling, rack space, firmware and driver maintenance, spares, and the engineering attention required to integrate it safely. In a small team, that attention is scarce and competes directly with reliability work and product delivery. Improving utilisation through scheduling, request batching, model routing, quantisation, caching, and better observability has its own cost, but it is mostly engineering time and can be staged. It also tends to produce reusable operational capability rather than a fixed asset that depreciates. The opportunity cost of buying now is therefore not just the capital; it is the chance to learn where our actual bottleneck sits and to avoid adding idle capacity that looks like progress but does not improve service.\n\nLatency and reliability need to be treated together. Better utilisation can reduce latency by reducing queueing and by routing requests to appropriate models or hardware, but aggressive batching or overcommitment can increase tail latency and create noisy-neighbour effects. Additional hardware can reduce latency when compute is genuinely saturated, but it can also add failure domains, power and cooling constraints, and operational complexity. Reliability improves when we have measured headroom and clear saturation signals; it degrades when we run existing equipment hot without observability or when we assume new hardware will solve a scheduling problem. We should define service-level objectives for latency and availability, measure queue time, utilisation, error rates, and headroom, and then decide whether the bottleneck is capacity or coordination.\n\nReversibility strongly favours the utilisation path first. Software changes such as scheduler policies, batching limits, routing rules, and caching can be rolled back or tuned quickly. A hardware purchase is largely irreversible: lead times, sunk cost, power provisioning, and integration work make it hard to undo if demand forecasts are wrong. That matters because our demand forecast is uncertain. We do not yet know whether current low utilisation reflects insufficient demand, poor scheduling, model inefficiency, or a mix of all three. We also do not know how quickly workload mix will change or whether vendor lead times will improve. A staged approach lets us buy information before we buy capacity.\n\nMy recommendation is to fund a bounded utilisation programme of roughly six weeks, with a named owner and a decision gate. The programme should establish accurate utilisation and latency baselines, implement SLO-aware scheduling and batching, evaluate model routing and quantisation where quality permits, and quantify achievable headroom under realistic load. At the gate, we should expand hardware only if we can show sustained demand that cannot be met within latency and reliability targets after reasonable utilisation improvements, and if the incremental capacity has a credible payback against the cost of power, space, and operations. If the gate is not met, we should keep improving utilisation and reserve spend for reliability spares or refresh rather than expansion. If it is met, we should buy a modest, well-justified increment rather than a large speculative fleet.\n\nThe uncertainty is real, and I do not claim that utilisation work will always be sufficient. Some workloads are compute-bound and will need more accelerators; some latency targets may require dedicated capacity. But we should make that purchase on evidence, not on the discomfort of low utilisation. The reversible, lower-cost step is to improve utilisation first, measure the result, and let the decision gate tell us whether additional local AI hardware is genuinely the next best use of our limited capital and engineering time.", "output_characters": 4613, "output_sha256": "0bbde7bdb40f4409d9ec5a18027d756109922b1f6da576aa75e326af1cbcbc34", "prompt_tokens": 136, "reasoning_characters": 16200, "reasoning_sha256": "c8c4d03c541f1f64256e2ea6c08f8e3d01905c92e54c5f4eee7187b9d41e46a8", "response_tail_seconds": 0.030219, "run": 1, "stream_done_marker": true, "stream_sha256": "f383f3868609aacfaca720d3c26ae9436e630717e9864222514f6d132429f70a", "time_to_first_visible_seconds": 42.030257, "time_to_last_output_seconds": 48.057074, "ttft_seconds": 0.372386, "wall_seconds": 48.087293 }, { "cached_prompt_tokens": 0, "completion_tokens": 1907, "completion_validation": { "valid": true }, "decode_seconds": 27.764922, "decode_tokens_per_second": 68.648, "finish_reason": "stop", "measured_sse_events": 904, "output": "**MEMORANDUM**\n\n**To:** AI Platform Team \n**From:** Engineering \n**Date:** [date] \n**Subject:** Capacity decision: buy more local AI hardware or improve utilisation\n\nI recommend that we do not buy additional local AI hardware yet. Instead, we should commit to a bounded utilisation programme with explicit performance and reliability gates, and reserve any purchase for the case where measured demand still exceeds capacity after those gates have been met. This is not a rejection of hardware expansion; it is a sequencing decision. Buying more capacity before we understand why existing capacity is underused risks spending capital to solve a scheduling and operations problem.\n\nThe cost case is straightforward but easy to misread. Local accelerators carry capital cost, power and cooling, rack space, maintenance, and depreciation, and those costs persist whether the hardware is busy or idle. If our current fleet has headroom lost to fragmented reservations, oversized model replicas, poor batching, or workloads pinned to specific devices, additional hardware will not fix that fragmentation; it will simply add more devices to underutilise. Improving utilisation lowers cost per inference and defers capital expenditure, but it also consumes engineering time. That time has an opportunity cost, because the same engineers could be improving reliability, reducing latency, or shipping product. The question is therefore not whether utilisation work is free, but whether it is cheaper and more reversible than buying capacity.\n\nLatency is the main reason utilisation improvements can fail. Batching, queueing, model routing, caching, and quantisation can raise throughput, but they can also increase tail latency if queues grow or if low-priority work competes with latency-sensitive requests. We should not optimise utilisation without service-level objectives for p95 and p99 latency, priority queues, and admission control. Buying hardware is a direct latency intervention, but it is an expensive one, and it can mask inefficiency by adding capacity rather than improving scheduling. The useful comparison is cost per request that meets the latency objective, not raw throughput or nominal utilisation.\n\nReliability cuts both ways. More hardware adds failure modes, power draw, firmware and driver complexity, and operational burden. Utilisation improvements can improve reliability by making scheduling, observability, and failover more explicit, but aggressive consolidation can create hotspots and single points of failure. We should target a safe operating range with burst headroom rather than chase one hundred percent utilisation. The reliability uncertainty is real: we do not yet know how our workload mix will behave under tighter scheduling, and we should measure error rates, queue waits, and recovery behaviour before treating utilisation gains as durable.\n\nOpportunity cost and reversibility are the decisive considerations. Hardware purchases have long lead times, depreciate quickly in this market, and can lock us into an architecture before our workload and model mix stabilise. Utilisation changes are comparatively reversible: we can adjust scheduling, routing, batching, and admission control, measure the result, and roll back. That reversibility is valuable under uncertainty about demand forecasts, future model sizes, and the hardware roadmap. Delaying a purchase does carry risk if demand grows faster than we expect, so the utilisation programme needs a clear exit condition rather than becoming an open-ended optimisation project.\n\nI propose a four-to-six week sprint that instruments per-device and per-queue utilisation, queue wait, throughput, error rates, and cost per request, then applies scheduling, batching, model routing, caching, and safe quantisation where the measurements justify it. We should define latency and reliability gates in advance. If, after the sprint, realistic load still produces p95 queue waits above target while safe utilisation is consistently saturated, we should buy targeted capacity. If not, we should continue improving utilisation and revisit the decision monthly. My recommendation is therefore to improve utilisation first, with a pre-agreed trigger for purchase. I acknowledge that this could be wrong if demand spikes or if new model sizes outstrip our current fleet, but the reversible path is the better default while those uncertainties remain unresolved.", "output_characters": 4440, "output_sha256": "11de503444f28e85523b84013606b095bfbc5fcb4d644977933e79d9e69ca52d", "prompt_tokens": 139, "reasoning_characters": 5642, "reasoning_sha256": "c9fc3e7ece59629c1a2a3b06fda047d706fba8da53da3352959fbed39ce67343", "response_tail_seconds": 0.000386, "run": 2, "stream_done_marker": true, "stream_sha256": "e85f19387d81389ed1773f0ff39be1045e0738e128541a9c077ee84e64186bae", "time_to_first_visible_seconds": 17.070894, "time_to_last_output_seconds": 28.132568, "ttft_seconds": 0.367646, "wall_seconds": 28.132954 }, { "cached_prompt_tokens": 0, "completion_tokens": 4933, "completion_validation": { "valid": true }, "decode_seconds": 55.369689, "decode_tokens_per_second": 89.074, "finish_reason": "stop", "measured_sse_events": 1668, "output": "Internal Engineering Memo\nSubject: Local AI capacity\u2014buy more hardware or improve utilisation?\nTo: Engineering leadership\nFrom: Infrastructure engineering\n\nThis memo addresses whether we should buy additional local AI hardware or first improve utilisation of the hardware we already own. The decision matters because local capacity is capital-intensive and operationally demanding, and because a small team has limited engineering and operations bandwidth. My recommendation is to run a bounded utilisation-first program and defer additional hardware purchase until we have evidence that we are genuinely capacity-bound after software and scheduling improvements. I would preserve the option to buy by pre-qualifying configurations and vendor terms, but not commit capital yet. The question is not whether local hardware has value, but whether additional capacity is the highest-return next investment.\n\nThe cost case is not simply the purchase price of accelerators. Additional local hardware brings power and cooling load, rack space, networking, spares, support contracts, depreciation, and the engineering time to integrate and operate it. Those costs are real even when the hardware sits idle, and idle capacity is especially expensive in a small deployment. Improving utilisation also costs engineering time, but that time is more granular and can be stopped if returns diminish. The opportunity cost cuts both ways: capital spent on more nodes cannot fund reliability spares, storage, network upgrades, or product work, while engineering time spent tuning schedulers cannot be spent on features. The right comparison is therefore marginal throughput and latency per dollar and per engineering week, not raw nameplate capacity.\n\nLatency is the main reason utilisation work must be done carefully. Higher utilisation often means larger batches, deeper queues, cache pressure, and less headroom. Continuous batching and KV-cache reuse can raise throughput substantially, but they can also worsen tail latency if admission control and scheduling are weak. Model routing to smaller models and quantization can reduce cost and latency for some requests, but they change quality and must be evaluated per workload. Additional hardware can reduce queueing delay and tail latency, but only if compute, memory bandwidth, or KV-cache capacity is the actual bottleneck. If the bottleneck is scheduling, model choice, or inefficient serving, more hardware will raise cost without fixing the user experience. We should measure effective throughput, queue depth, p95 and p99 latency, memory pressure, and errors under realistic traffic, not just average GPU utilisation.\n\nReliability also argues for caution. More local nodes add failure domains, power and cooling constraints, firmware and driver variance, and operational burden. A small team can absorb software-level improvements\u2014better admission control, health checks, failover, and capacity reservation\u2014more safely than it can absorb a larger fleet. At the same time, overcommitting existing hardware can create out-of-memory failures, timeout storms, and noisy-neighbour effects. The utilisation program must therefore preserve explicit headroom and reliability SLOs rather than chase a utilisation number. If we buy hardware, we should buy enough redundancy and spares to avoid turning a capacity improvement into a reliability regression.\n\nReversibility is decisive. Software and configuration changes are comparatively reversible: we can roll back a scheduler, revert a quantization policy, or adjust admission thresholds. Hardware procurement is not. Lead times, integration work, depreciation, and resale losses make a purchase hard to undo, and a small team may be stuck operating capacity it does not need. That asymmetry favours proving demand first. The uncertainty is genuine: we do not know how much demand will grow, how much software efficiency we can extract, or how quickly vendor lead times will change. A staged approach handles that uncertainty better than a binary bet.\n\nI recommend a six-to-eight-week utilisation effort with clear gates. We should establish baseline throughput, latency, reliability, and cost per request; implement the highest-return scheduling and serving improvements; and then re-measure under peak load. If sustained peak utilisation remains high after those changes, queueing latency breaches SLOs, memory pressure is persistent, and the marginal cost of another node is justified by demand, we should buy. If not, we should keep improving utilisation and revisit the purchase quarterly. The gates should be agreed with product and operations so that the decision is not driven by a single metric. This is not a rejection of hardware; it is a way to buy it when the evidence is strong and the commitment is reversible enough to defend.", "output_characters": 4829, "output_sha256": "c7cc3b1409dd8b77e9eaf4ad6476014c79980197e09d6dc8fae66830f605f06e", "prompt_tokens": 141, "reasoning_characters": 18908, "reasoning_sha256": "d746773972692524c6af20c09a653351fcc7edded9cf063c6494fc46ab51b1c1", "response_tail_seconds": 0.040529, "run": 3, "stream_done_marker": true, "stream_sha256": "eedd5cc5fdbfc4d469c2503988a1cb8e88d3a39e2ad5a79c3f3308635bfe04ec", "time_to_first_visible_seconds": 49.171631, "time_to_last_output_seconds": 55.735951, "ttft_seconds": 0.366262, "wall_seconds": 55.77648 }, { "cached_prompt_tokens": 0, "completion_tokens": 1436, "completion_validation": { "valid": true }, "decode_seconds": 21.585079, "decode_tokens_per_second": 66.481, "finish_reason": "stop", "measured_sse_events": 728, "output": "**Internal Engineering Memo**\n\n**To:** Platform Engineering and AI Infrastructure Team \n**From:** Engineering Infrastructure Review \n**Subject:** Local AI capacity: optimise before expanding\n\nThe immediate question is whether to buy more local AI hardware or to improve utilisation of the hardware we already own. My recommendation is to treat additional procurement as a second-stage decision, not the default response. We should first fund a focused utilisation programme with clear instrumentation, scheduling improvements, and a capped engineering budget, then buy incrementally only if sustained demand remains unmet after those changes. This is not a claim that we will never need more hardware. It is a claim that we do not yet have enough evidence to justify a large irreversible capital commitment, and that the cheaper and more reversible work should come first.\n\nCost is the obvious starting point, but the relevant cost is not just the purchase price of additional accelerators. Local capacity carries power, cooling, rack space, depreciation, spare-parts inventory, firmware and driver maintenance, and the engineering time required to keep a small cluster healthy. A small team pays a disproportionate fixed cost for every additional node, and that cost persists even when demand is spiky or model mix changes. Improving utilisation also has a cost, chiefly engineering time, but it is largely reversible and it produces information we need anyway. If we buy first, we risk capitalising a forecast that may be wrong and then discovering that the real constraint was queueing policy, memory fragmentation, or poor model routing rather than raw compute.\n\nLatency deserves careful treatment because \u201cutilisation\u201d can be a misleading target. Pushing average GPU utilisation higher by increasing batch sizes may improve throughput while worsening interactive tail latency. Conversely, reserving capacity for low-concurrency requests can look wasteful on a dashboard while protecting user experience. We should measure queue wait, prefill and decode latency separately, cache hit rates, memory headroom, and the distribution of request sizes, not just aggregate utilisation. The goal is effective utilisation under our latency objectives, not maximum occupancy. If the data shows that interactive traffic is already meeting its service-level targets and that batch work is being scheduled into otherwise idle windows, additional hardware may not improve the user-facing experience at all.\n\nReliability cuts both ways. More local nodes can add redundancy, but a small cluster with heterogeneous parts and limited spares can also become more fragile. Local hardware failures are our responsibility, and recovery time depends on spare inventory, driver compatibility, and operational familiarity. A utilisation programme should include health checks, graceful draining, and clear fallback paths. Where reliability requirements exceed what a small local cluster can economically provide, a modest cloud overflow or burst arrangement may be more sensible than buying standby capacity that sits idle most of the year. That overflow is also a reversibility mechanism: it lets us absorb demand peaks without committing to permanent capacity.\n\nOpportunity cost is perhaps the strongest argument for sequencing. Every engineering hour spent on procurement, racking, burn-in, and cluster operations is an hour not spent on scheduling, quantization, model routing, caching, or workload isolation. Those software changes often deliver more usable capacity per dollar than a new node, and they can be evaluated quickly. A small team should not split its attention across a capital purchase and a utilisation programme simultaneously unless demand clearly justifies both.\n\nReversibility should shape the procurement decision itself. If we eventually buy, we should prefer incremental, modular additions, short leases where available, and cloud overflow as a pressure valve. That preserves the ability to change course if model architectures, memory requirements, or demand patterns shift. Large upfront purchases are difficult to undo and can lock us into a hardware generation just before a better option becomes available.\n\nI therefore recommend a time-boxed utilisation phase with explicit success criteria: sustained queue wait above target, batch work unable to fit into idle windows, memory headroom consistently exhausted, and no further software gains available at reasonable engineering cost. If those conditions hold after the programme, we should buy incrementally rather than in a single large commitment. The main uncertainty is demand forecasting and workload mix; we should not pretend to know them precisely. The decision rule should be evidence-based, reversible where possible, and biased toward improving what we own before adding permanent capacity.", "output_characters": 4864, "output_sha256": "9793da164e2d2ada71c6f0b6f8c1d21177238f7706665e1d0e8b6f2408f3a97a", "prompt_tokens": 135, "reasoning_characters": 2881, "reasoning_sha256": "7b9ebf091651de963c925ff926cdc994ea34401e62a69cd537b673870ae88590", "response_tail_seconds": 0.000447, "run": 4, "stream_done_marker": true, "stream_sha256": "d21f044eebc63585b35a05b89ceaca10d665c9c2b018b064b7ea91b1b66f8647", "time_to_first_visible_seconds": 8.87074, "time_to_last_output_seconds": 21.9535, "ttft_seconds": 0.368421, "wall_seconds": 21.953947 }, { "cached_prompt_tokens": 0, "completion_tokens": 3104, "completion_validation": { "valid": true }, "decode_seconds": 38.392091, "decode_tokens_per_second": 80.824, "finish_reason": "stop", "measured_sse_events": 1196, "output": "Internal Engineering Memo\n\nTo the engineering leadership team, from the platform engineering group, subject: local AI capacity decision.\n\nWe need to decide whether to buy more local AI hardware or to improve utilisation of the hardware we already own. My recommendation is to make utilisation the primary investment for the next quarter and to defer additional hardware purchases until we have measured evidence that optimised capacity cannot meet demand. This is not a rejection of hardware; it is a sequencing decision. We should set aside a capped contingency and define purchase triggers in advance, so we can move quickly if the data justifies it.\n\nThe cost case is asymmetric. Additional local AI hardware carries capital cost, but also rack space, power, cooling, spare parts, maintenance, and engineering attention. A small team feels those operational costs disproportionately. Utilisation work costs mostly engineering time and some software changes, and it can often be done incrementally. The risk is that engineering time is scarce and could be spent on product or model work. That opportunity cost is real, so the utilisation effort should be narrowly scoped to high-leverage changes: observability of queue depth and tail latency, better batching and scheduling, model routing, quantization where quality allows, cache reuse, admission control, and warm pools for latency-sensitive workloads. We should not attempt a broad platform rewrite. We should measure whether these changes move the bottleneck.\n\nLatency is the strongest argument for caution about simply adding capacity. Low average utilisation does not guarantee good tail latency. Cold starts, model loading, memory fragmentation, inefficient batching, and poor routing can produce high p95 or p99 latency even when GPUs look idle. More hardware can reduce queueing, but if the real problem is scheduling or memory pressure, it adds idle capacity and complexity without fixing the user experience. Conversely, utilisation improvements can reduce latency by increasing effective concurrency and keeping hot models warm. We should track p95 latency, queue depth, error rates, and cost per token alongside utilisation, because a single utilisation percentage is misleading.\n\nReliability also cuts both ways. More hardware can provide redundancy and headroom, but it also creates more failure domains, more firmware and driver variance, and more operational burden for a small team. Utilisation improvements can improve reliability if they include admission control, graceful degradation, and explicit headroom rather than running everything at the edge of saturation. The goal is not maximum utilisation; it is predictable service with enough reserve to absorb bursts and failures. We should define a reliability margin and protect it.\n\nReversibility favours the utilisation-first path. Software and configuration changes can be rolled back, measured, and adjusted. Hardware purchases are much less reversible: depreciation is real, resale value is uncertain, and the model and runtime landscape may shift before the hardware is fully amortised. That uncertainty matters because our demand forecast and model roadmap are not stable. We do not know whether future workloads will reward more memory, more compute, different accelerators, or simply better scheduling. Buying now locks in an assumption. Improving utilisation preserves options and gives us better data for the next decision.\n\nMy recommendation is therefore to run a focused utilisation programme for one quarter, with a small capped budget and clear success metrics. We should pre-approve a hardware contingency but not spend it unless triggers are met: sustained utilisation above a defined threshold after optimisation, p95 latency consistently outside the service target, queue depth that cannot be reduced by scheduling, or reliability headroom below the agreed margin. If those triggers fire, we should buy incremental, modular capacity that is easy to redeploy rather than a large fixed commitment. If they do not, we will have saved capital and learned where the real bottleneck is. The main uncertainty is demand growth; if a known product launch will double inference load, we should revisit the trigger earlier. But absent that evidence, buying more hardware now is the less reversible and less informed choice. We should improve utilisation first, measure honestly, and let the data decide.", "output_characters": 4436, "output_sha256": "fb95740c1389e4eb872e2a619ba4765a3e7fb97e09937fe49b7bbe3edc347808", "prompt_tokens": 137, "reasoning_characters": 11723, "reasoning_sha256": "391cab8d8ae23aab3851640ae810266a49d03e39557b379bae48665ec62e1844", "response_tail_seconds": 0.029772, "run": 5, "stream_done_marker": true, "stream_sha256": "b60a19935df7e9d46ae61fb24968c0264b0cd6caabd2d0131cb221ad9995c1fc", "time_to_first_visible_seconds": 32.954972, "time_to_last_output_seconds": 38.76906, "ttft_seconds": 0.376969, "wall_seconds": 38.798832 } ], "time_to_last_output_seconds": { "maximum": 55.736, "median": 38.769, "minimum": 21.953, "p90": 55.736 }, "ttft_seconds": { "maximum": 0.377, "median": 0.368, "minimum": 0.366, "p90": 0.377 }, "wall_seconds": { "maximum": 55.776, "median": 38.799, "minimum": 21.954, "p90": 55.776 } }, "structured": { "completion_gate": { "passed": 5, "total": 5 }, "decode_tokens_per_second": { "maximum": 130.788, "median": 130.183, "minimum": 128.125, "p90": 130.788 }, "pooled_decode_tokens_per_second": 129.804, "runs": [ { "cached_prompt_tokens": 0, "completion_tokens": 724, "completion_validation": { "valid": true }, "decode_seconds": 5.528012, "decode_tokens_per_second": 130.788, "finish_reason": "stop", "measured_sse_events": 141, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 127, "reasoning_characters": 710, "reasoning_sha256": "bf7508c2df6053160ce193489f7c5dab73272c94c2d0e7295631fa2e3f07d4a4", "response_tail_seconds": 0.000186, "run": 1, "stream_done_marker": true, "stream_sha256": "0178d38b41016cbe95bec8d19b79987c445b1b94451a099b2fbd96b74a100df1", "time_to_first_visible_seconds": 2.897387, "time_to_last_output_seconds": 5.866814, "ttft_seconds": 0.338803, "wall_seconds": 5.867001 }, { "cached_prompt_tokens": 0, "completion_tokens": 726, "completion_validation": { "valid": true }, "decode_seconds": 5.569104, "decode_tokens_per_second": 130.183, "finish_reason": "stop", "measured_sse_events": 142, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 127, "reasoning_characters": 720, "reasoning_sha256": "48a5f1f741f41913996e5954057c15c79f78857c3831e5ee7043444644c16eb0", "response_tail_seconds": 0.000193, "run": 2, "stream_done_marker": true, "stream_sha256": "ed7bba095d96851bcfa5d5e9521b5742ddcfb9063cd43c84a4329041d5009c8f", "time_to_first_visible_seconds": 2.938745, "time_to_last_output_seconds": 5.90646, "ttft_seconds": 0.337356, "wall_seconds": 5.906653 }, { "cached_prompt_tokens": 0, "completion_tokens": 724, "completion_validation": { "valid": true }, "decode_seconds": 5.551678, "decode_tokens_per_second": 130.231, "finish_reason": "stop", "measured_sse_events": 140, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 124, "reasoning_characters": 710, "reasoning_sha256": "da63441f449cb5428b30dcb8e8b52286aaae7142fe70d7bfdd55b4ac6391a3c5", "response_tail_seconds": 0.000146, "run": 3, "stream_done_marker": true, "stream_sha256": "e98b0b7546cdc0906d80f9dd4f37273f2a900f13a17512422e4fab177dab6d8f", "time_to_first_visible_seconds": 2.878316, "time_to_last_output_seconds": 5.879262, "ttft_seconds": 0.327584, "wall_seconds": 5.879408 }, { "cached_prompt_tokens": 0, "completion_tokens": 726, "completion_validation": { "valid": true }, "decode_seconds": 5.658526, "decode_tokens_per_second": 128.125, "finish_reason": "stop", "measured_sse_events": 143, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 123, "reasoning_characters": 715, "reasoning_sha256": "a8d2fe82e6c6a53924d1c6ff187f0daf09633e3b38936945a0e943d9bdc0c55a", "response_tail_seconds": 0.000283, "run": 4, "stream_done_marker": true, "stream_sha256": "c95138f461cb41117143d03b0884f64abfc0c6c15a922bfac31c242fc95fa7de", "time_to_first_visible_seconds": 2.988597, "time_to_last_output_seconds": 5.994933, "ttft_seconds": 0.336407, "wall_seconds": 5.995216 }, { "cached_prompt_tokens": 0, "completion_tokens": 712, "completion_validation": { "valid": true }, "decode_seconds": 5.480711, "decode_tokens_per_second": 129.728, "finish_reason": "stop", "measured_sse_events": 138, "output": "[{\"index\":1,\"square\":1},{\"index\":2,\"square\":4},{\"index\":3,\"square\":9},{\"index\":4,\"square\":16},{\"index\":5,\"square\":25},{\"index\":6,\"square\":36},{\"index\":7,\"square\":49},{\"index\":8,\"square\":64},{\"index\":9,\"square\":81},{\"index\":10,\"square\":100},{\"index\":11,\"square\":121},{\"index\":12,\"square\":144},{\"index\":13,\"square\":169},{\"index\":14,\"square\":196},{\"index\":15,\"square\":225},{\"index\":16,\"square\":256},{\"index\":17,\"square\":289},{\"index\":18,\"square\":324},{\"index\":19,\"square\":361},{\"index\":20,\"square\":400},{\"index\":21,\"square\":441},{\"index\":22,\"square\":484},{\"index\":23,\"square\":529},{\"index\":24,\"square\":576},{\"index\":25,\"square\":625},{\"index\":26,\"square\":676},{\"index\":27,\"square\":729},{\"index\":28,\"square\":784},{\"index\":29,\"square\":841},{\"index\":30,\"square\":900},{\"index\":31,\"square\":961},{\"index\":32,\"square\":1024},{\"index\":33,\"square\":1089},{\"index\":34,\"square\":1156},{\"index\":35,\"square\":1225},{\"index\":36,\"square\":1296},{\"index\":37,\"square\":1369},{\"index\":38,\"square\":1444},{\"index\":39,\"square\":1521},{\"index\":40,\"square\":1600},{\"index\":41,\"square\":1681},{\"index\":42,\"square\":1764},{\"index\":43,\"square\":1849},{\"index\":44,\"square\":1936},{\"index\":45,\"square\":2025},{\"index\":46,\"square\":2116},{\"index\":47,\"square\":2209},{\"index\":48,\"square\":2304},{\"index\":49,\"square\":2401},{\"index\":50,\"square\":2500}]", "output_characters": 1299, "output_sha256": "654a5359e59bad57071f454922f88d55d393a17d6eec293ae79ea185c3cd7453", "prompt_tokens": 126, "reasoning_characters": 649, "reasoning_sha256": "808c1d99a98225ecbbb5351ddeb5ac80e8cf078139dac607243a694ec954d6ee", "response_tail_seconds": 0.033841, "run": 5, "stream_done_marker": true, "stream_sha256": "82c4d8f102c89b0cc17f8f47750b474c066ecfdedc2bd1c7446bc96ae1330c02", "time_to_first_visible_seconds": 2.813113, "time_to_last_output_seconds": 5.81497, "ttft_seconds": 0.334259, "wall_seconds": 5.848811 } ], "time_to_last_output_seconds": { "maximum": 5.995, "median": 5.879, "minimum": 5.815, "p90": 5.995 }, "ttft_seconds": { "maximum": 0.339, "median": 0.336, "minimum": 0.328, "p90": 0.339 }, "wall_seconds": { "maximum": 5.995, "median": 5.879, "minimum": 5.849, "p90": 5.995 } } }, "prefill": { "32768": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2504.481, "median": 2499.546, "minimum": 2475.994, "p90": 2504.481 }, "runs": [ { "cache_salt_sha256": "f6f5f584e224f455369a67c076fc9904267867b6fbbf946a2fc46f6cc997febb", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.077563, "decode_tokens_per_second": 90.25, "effective_prefill_tokens_per_second": 2475.994, "finish_reason": "length", "measured_sse_events": 3, "output": " action. The repository contains a service", "output_characters": 42, "output_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000201, "run": 1, "stream_done_marker": true, "stream_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "time_to_first_visible_seconds": 13.234281, "time_to_last_output_seconds": 13.311844, "ttft_seconds": 13.234281, "wall_seconds": 13.312045 }, { "cache_salt_sha256": "bbf3da847ebe2edb4cd7488b33c556622082084560f4661fb32fc0d684816df0", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.077809, "decode_tokens_per_second": 89.964, "effective_prefill_tokens_per_second": 2499.546, "finish_reason": "length", "measured_sse_events": 3, "output": ". The repository contains a service,", "output_characters": 36, "output_sha256": "6b7a31399cbcdf5c725277ca09708289bb776a5e506696b6d065d920946f0841", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000204, "run": 2, "stream_done_marker": true, "stream_sha256": "6b7a31399cbcdf5c725277ca09708289bb776a5e506696b6d065d920946f0841", "time_to_first_visible_seconds": 13.109582, "time_to_last_output_seconds": 13.187391, "ttft_seconds": 13.109582, "wall_seconds": 13.187595 }, { "cache_salt_sha256": "f5a92ff27943a4146bbb644390b6dce670be0f8c1128933d2707d1f8cd6582a5", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.078681, "decode_tokens_per_second": 88.967, "effective_prefill_tokens_per_second": 2504.481, "finish_reason": "length", "measured_sse_events": 3, "output": " action. The repository contains a service", "output_characters": 42, "output_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000171, "run": 3, "stream_done_marker": true, "stream_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "time_to_first_visible_seconds": 13.083748, "time_to_last_output_seconds": 13.162429, "ttft_seconds": 13.083748, "wall_seconds": 13.1626 } ], "ttft_seconds": { "maximum": 13.234, "median": 13.11, "minimum": 13.084, "p90": 13.234 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 32827.714, "median": 32076.526, "minimum": 31948.738, "p90": 32827.714 }, "runs": [ { "cache_salt_sha256": "f6f5f584e224f455369a67c076fc9904267867b6fbbf946a2fc46f6cc997febb", "cached_prompt_tokens": 30720, "completion_tokens": 8, "decode_seconds": 0.078604, "decode_tokens_per_second": 89.053, "effective_prefill_tokens_per_second": 31948.738, "finish_reason": "length", "measured_sse_events": 3, "output": " action. The repository contains a service", "output_characters": 42, "output_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000334, "run": 1, "stream_done_marker": true, "stream_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "time_to_first_visible_seconds": 1.025643, "time_to_last_output_seconds": 1.104247, "ttft_seconds": 1.025643, "wall_seconds": 1.104581 }, { "cache_salt_sha256": "bbf3da847ebe2edb4cd7488b33c556622082084560f4661fb32fc0d684816df0", "cached_prompt_tokens": 30720, "completion_tokens": 8, "decode_seconds": 0.078659, "decode_tokens_per_second": 88.992, "effective_prefill_tokens_per_second": 32076.526, "finish_reason": "length", "measured_sse_events": 3, "output": ". The repository contains a service,", "output_characters": 36, "output_sha256": "6b7a31399cbcdf5c725277ca09708289bb776a5e506696b6d065d920946f0841", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000252, "run": 2, "stream_done_marker": true, "stream_sha256": "6b7a31399cbcdf5c725277ca09708289bb776a5e506696b6d065d920946f0841", "time_to_first_visible_seconds": 1.021557, "time_to_last_output_seconds": 1.100216, "ttft_seconds": 1.021557, "wall_seconds": 1.100468 }, { "cache_salt_sha256": "f5a92ff27943a4146bbb644390b6dce670be0f8c1128933d2707d1f8cd6582a5", "cached_prompt_tokens": 30720, "completion_tokens": 8, "decode_seconds": 0.077881, "decode_tokens_per_second": 89.881, "effective_prefill_tokens_per_second": 32827.714, "finish_reason": "length", "measured_sse_events": 3, "output": " action. The repository contains a service", "output_characters": 42, "output_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "prompt_tokens": 32768, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 32768, "response_tail_seconds": 0.000176, "run": 3, "stream_done_marker": true, "stream_sha256": "b5a7611b4f1762cd19a22471acbe46b2c959f51e351ddcc2332e20186a3f3dd8", "time_to_first_visible_seconds": 0.998181, "time_to_last_output_seconds": 1.076062, "ttft_seconds": 0.998181, "wall_seconds": 1.076238 } ], "ttft_seconds": { "maximum": 1.026, "median": 1.022, "minimum": 0.998, "p90": 1.026 } } }, "65536": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2497.591, "median": 2488.44, "minimum": 2482.813, "p90": 2497.591 }, "runs": [ { "cache_salt_sha256": "cce5891a7a2b9ef58e5e11f299c6b42aa1939aa987005dc3e185b7697c20927b", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.079436, "decode_tokens_per_second": 88.121, "effective_prefill_tokens_per_second": 2482.813, "finish_reason": "length", "measured_sse_events": 3, "output": " contains a service, tests, documentation,", "output_characters": 42, "output_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000148, "run": 1, "stream_done_marker": true, "stream_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "time_to_first_visible_seconds": 26.395865, "time_to_last_output_seconds": 26.475301, "ttft_seconds": 26.395865, "wall_seconds": 26.475449 }, { "cache_salt_sha256": "e7bb9262069a043d9e542d5a5d481e1317a18f9cf6c1cb17d34fc5ef6f4cb606", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.080365, "decode_tokens_per_second": 87.103, "effective_prefill_tokens_per_second": 2488.44, "finish_reason": "length", "measured_sse_events": 3, "output": " the next action. The repository contains", "output_characters": 41, "output_sha256": "d0ecfe654cea47153ea77ef98b0550d792072bcb3a007c5d9fe0f911301fcbb7", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000137, "run": 2, "stream_done_marker": true, "stream_sha256": "d0ecfe654cea47153ea77ef98b0550d792072bcb3a007c5d9fe0f911301fcbb7", "time_to_first_visible_seconds": 26.33618, "time_to_last_output_seconds": 26.416545, "ttft_seconds": 26.33618, "wall_seconds": 26.416682 }, { "cache_salt_sha256": "9d0f170d20dd09a560fa8168bbcb97c4246c9aa145130a9cb2764b1b2cdca1d1", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.080371, "decode_tokens_per_second": 87.096, "effective_prefill_tokens_per_second": 2497.591, "finish_reason": "length", "measured_sse_events": 3, "output": " next action. The repository contains a", "output_characters": 39, "output_sha256": "dc5bcdac54c41a22c11643f735434b20c9079448f0b90a92b5234f70a0094140", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000149, "run": 3, "stream_done_marker": true, "stream_sha256": "dc5bcdac54c41a22c11643f735434b20c9079448f0b90a92b5234f70a0094140", "time_to_first_visible_seconds": 26.239687, "time_to_last_output_seconds": 26.320058, "ttft_seconds": 26.239687, "wall_seconds": 26.320206 } ], "ttft_seconds": { "maximum": 26.396, "median": 26.336, "minimum": 26.24, "p90": 26.396 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 63720.104, "median": 62744.678, "minimum": 62215.851, "p90": 63720.104 }, "runs": [ { "cache_salt_sha256": "cce5891a7a2b9ef58e5e11f299c6b42aa1939aa987005dc3e185b7697c20927b", "cached_prompt_tokens": 63488, "completion_tokens": 8, "decode_seconds": 0.079979, "decode_tokens_per_second": 87.523, "effective_prefill_tokens_per_second": 63720.104, "finish_reason": "length", "measured_sse_events": 3, "output": " contains a service, tests, documentation,", "output_characters": 42, "output_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000143, "run": 1, "stream_done_marker": true, "stream_sha256": "2be23cdd1424b5a316110cb407a1179a749697fa53d20932faccec5f35408b4b", "time_to_first_visible_seconds": 1.028498, "time_to_last_output_seconds": 1.108477, "ttft_seconds": 1.028498, "wall_seconds": 1.10862 }, { "cache_salt_sha256": "e7bb9262069a043d9e542d5a5d481e1317a18f9cf6c1cb17d34fc5ef6f4cb606", "cached_prompt_tokens": 63488, "completion_tokens": 8, "decode_seconds": 0.086285, "decode_tokens_per_second": 81.126, "effective_prefill_tokens_per_second": 62215.851, "finish_reason": "length", "measured_sse_events": 3, "output": " the next action. The repository contains", "output_characters": 41, "output_sha256": "d0ecfe654cea47153ea77ef98b0550d792072bcb3a007c5d9fe0f911301fcbb7", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000161, "run": 2, "stream_done_marker": true, "stream_sha256": "d0ecfe654cea47153ea77ef98b0550d792072bcb3a007c5d9fe0f911301fcbb7", "time_to_first_visible_seconds": 1.053365, "time_to_last_output_seconds": 1.13965, "ttft_seconds": 1.053365, "wall_seconds": 1.139811 }, { "cache_salt_sha256": "9d0f170d20dd09a560fa8168bbcb97c4246c9aa145130a9cb2764b1b2cdca1d1", "cached_prompt_tokens": 63488, "completion_tokens": 8, "decode_seconds": 0.080382, "decode_tokens_per_second": 87.084, "effective_prefill_tokens_per_second": 62744.678, "finish_reason": "length", "measured_sse_events": 3, "output": " next action. The repository contains a", "output_characters": 39, "output_sha256": "dc5bcdac54c41a22c11643f735434b20c9079448f0b90a92b5234f70a0094140", "prompt_tokens": 65536, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 65536, "response_tail_seconds": 0.000147, "run": 3, "stream_done_marker": true, "stream_sha256": "dc5bcdac54c41a22c11643f735434b20c9079448f0b90a92b5234f70a0094140", "time_to_first_visible_seconds": 1.044487, "time_to_last_output_seconds": 1.124869, "ttft_seconds": 1.044487, "wall_seconds": 1.125016 } ], "ttft_seconds": { "maximum": 1.053, "median": 1.044, "minimum": 1.028, "p90": 1.053 } } }, "8192": { "cold": { "effective_prefill_tokens_per_second": { "maximum": 2412.822, "median": 2403.176, "minimum": 2340.84, "p90": 2412.822 }, "runs": [ { "cache_salt_sha256": "823be761f1f2b02f4c115c5e79fa33a370c1b46c50c0b34fb9c5e00027b996e3", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.075705, "decode_tokens_per_second": 92.464, "effective_prefill_tokens_per_second": 2412.822, "finish_reason": "length", "measured_sse_events": 3, "output": " observation records behaviour, evidence, constraints,", "output_characters": 54, "output_sha256": "64de6b20e549222d7456ec41b7e90d976a3186746092d94e04bb350c64dbcfd2", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000205, "run": 1, "stream_done_marker": true, "stream_sha256": "64de6b20e549222d7456ec41b7e90d976a3186746092d94e04bb350c64dbcfd2", "time_to_first_visible_seconds": 3.395194, "time_to_last_output_seconds": 3.470899, "ttft_seconds": 3.395194, "wall_seconds": 3.471104 }, { "cache_salt_sha256": "3a305ba0476421abae2079e2b9bbdff0d0991e8dcf3e37e27b43ed66df46226e", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.066691, "decode_tokens_per_second": 104.962, "effective_prefill_tokens_per_second": 2403.176, "finish_reason": "length", "measured_sse_events": 3, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000205, "run": 2, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 3.408822, "time_to_last_output_seconds": 3.475513, "ttft_seconds": 3.408822, "wall_seconds": 3.475718 }, { "cache_salt_sha256": "0da875054a39ccd5135dd136a4448b171bbedff92aa5472d12f2405e2878b92c", "cached_prompt_tokens": 0, "completion_tokens": 8, "decode_seconds": 0.076703, "decode_tokens_per_second": 91.261, "effective_prefill_tokens_per_second": 2340.84, "finish_reason": "length", "measured_sse_events": 3, "output": " Each observation records behaviour, evidence, constraints", "output_characters": 58, "output_sha256": "3f91e56380da40aa484278ca6ad46278f1415f32fb168c9a22b4e5fb220e39c0", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000202, "run": 3, "stream_done_marker": true, "stream_sha256": "3f91e56380da40aa484278ca6ad46278f1415f32fb168c9a22b4e5fb220e39c0", "time_to_first_visible_seconds": 3.499598, "time_to_last_output_seconds": 3.576301, "ttft_seconds": 3.499598, "wall_seconds": 3.576503 } ], "ttft_seconds": { "maximum": 3.5, "median": 3.409, "minimum": 3.395, "p90": 3.5 } }, "warm_replay": { "effective_prefill_tokens_per_second": { "maximum": 8367.517, "median": 8305.116, "minimum": 8238.888, "p90": 8367.517 }, "runs": [ { "cache_salt_sha256": "823be761f1f2b02f4c115c5e79fa33a370c1b46c50c0b34fb9c5e00027b996e3", "cached_prompt_tokens": 6144, "completion_tokens": 8, "decode_seconds": 0.07561, "decode_tokens_per_second": 92.581, "effective_prefill_tokens_per_second": 8305.116, "finish_reason": "length", "measured_sse_events": 3, "output": " observation records behaviour, evidence, constraints,", "output_characters": 54, "output_sha256": "64de6b20e549222d7456ec41b7e90d976a3186746092d94e04bb350c64dbcfd2", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000156, "run": 1, "stream_done_marker": true, "stream_sha256": "64de6b20e549222d7456ec41b7e90d976a3186746092d94e04bb350c64dbcfd2", "time_to_first_visible_seconds": 0.98638, "time_to_last_output_seconds": 1.06199, "ttft_seconds": 0.98638, "wall_seconds": 1.062147 }, { "cache_salt_sha256": "3a305ba0476421abae2079e2b9bbdff0d0991e8dcf3e37e27b43ed66df46226e", "cached_prompt_tokens": 6144, "completion_tokens": 8, "decode_seconds": 0.075109, "decode_tokens_per_second": 93.197, "effective_prefill_tokens_per_second": 8367.517, "finish_reason": "length", "measured_sse_events": 3, "output": ", evidence, constraints, and the next", "output_characters": 37, "output_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000162, "run": 2, "stream_done_marker": true, "stream_sha256": "f3550221b55f625769be944ebb28e9089be798e469bcacb77e28b60998b2b7ab", "time_to_first_visible_seconds": 0.979024, "time_to_last_output_seconds": 1.054134, "ttft_seconds": 0.979024, "wall_seconds": 1.054296 }, { "cache_salt_sha256": "0da875054a39ccd5135dd136a4448b171bbedff92aa5472d12f2405e2878b92c", "cached_prompt_tokens": 6144, "completion_tokens": 8, "decode_seconds": 0.075768, "decode_tokens_per_second": 92.387, "effective_prefill_tokens_per_second": 8238.888, "finish_reason": "length", "measured_sse_events": 3, "output": " Each observation records behaviour, evidence, constraints", "output_characters": 58, "output_sha256": "3f91e56380da40aa484278ca6ad46278f1415f32fb168c9a22b4e5fb220e39c0", "prompt_tokens": 8192, "reasoning_characters": 0, "reasoning_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requested_prompt_tokens": 8192, "response_tail_seconds": 0.000164, "run": 3, "stream_done_marker": true, "stream_sha256": "3f91e56380da40aa484278ca6ad46278f1415f32fb168c9a22b4e5fb220e39c0", "time_to_first_visible_seconds": 0.994309, "time_to_last_output_seconds": 1.070077, "ttft_seconds": 0.994309, "wall_seconds": 1.070241 } ], "ttft_seconds": { "maximum": 0.994, "median": 0.986, "minimum": 0.979, "p90": 0.994 } } } }, "protocol": { "prompts_sha256": "0c3ac4015813af6a1b967659bd2289990fde895a5e43ae065b894d1e1f909ecb", "prompts_version": "1.0.0", "repository_dirty": false, "repository_revision": "946394fa53db83a6865f6e3ae56594e1c299fb2a", "repository_source": "git-worktree", "repository_source_sha256": "f5503bd3fe66d96d42992fedb6cf0b47f1e7ba560359d455a6e8fc1c471261d7", "version": "1.3.0" }, "run": { "appliance": { "competing_traffic": "none intended: owner-gateway quiet window (only ALBERT+MINIME keys); client: FrankenNAS over 1 GbE LAN to the lane's rank-0 port (same path for every lane)", "context_limit": 1048576, "drafter": "model's own MTP", "hardware": "2x NVIDIA DGX Spark (GB10, 128 GB unified memory each), ascent-07 + ascent-08", "kv_cache_dtype": "engine default for this lane", "model": "bertholomus DeepSeek-V4.1-Flash-1017 EXL3 2.5 bpw", "model_revision": "local fingerprint 1bae0e10", "negotiated_link_speed": "200 Gb/s per CX7 function", "quantisation": "EXL3 2.5 bpw weights, BF16 activations", "scheduler": "lane as served in the bake-off, unchanged", "serving_engine": "TensorFold native (tensorfold-native, image nvcr.io/nvidia/pytorch:26.07-py3)", "serving_image": "see serving_engine", "topology": "TP2, one rank per Spark, RoCE over ConnectX-7" }, "comparison_id": "2026-10-two-sparks-four-models-v1", "finished_at": "2026-10-29T03:06:12.055847+00:00", "label": "bakeoff-ds", "model": "DeepSeek-V4.1-Flash-1017", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.43", "python": "3.11.15", "rigmark_release_check": { "ahead_by": 6, "behind_by": 0, "dirty": false, "prerelease": true, "release_url": "https://github.com/alexellis/rigmark/releases/tag/v0.0.4", "revision": "946394fa53db83a6865f6e3ae56594e1c299fb2a", "status": "ahead", "tag": "v0.0.4" }, "started_at": "2026-10-29T02:53:38.628542+00:00" }, "schema": 1, "settings": { "concurrency": [ 1, 2, 4 ], "concurrency_runs": 3, "concurrency_tokens": 256, "concurrency_workload": "code", "decode_tokens": 8192, "extra_body": {}, "prefill_cache_isolation": "random-salt-per-pair", "prefill_depths": [ 8192, 32768, 65536 ], "prefill_runs": 3, "runs": 5, "seed": 20260905, "temperature": 0.0, "top_p": 1.0 } }