{ "schema_version": 2, "fixture": false, "started_at": "2026-11-05T12:21:30.964Z", "finished_at": "2026-11-05T12:22:34.202Z", "endpoint": "https://api.excessionlabs.ai/v1", "runtime": "v26.12.1", "client_platform": "darwin/arm64", "concurrency": 1, "order": "Alternate which model goes first each round; same prompt for both in a round; rotate three prompts; no warmup or retries.", "samples_per_model": 20, "request_template": { "temperature": 0, "max_tokens": 512, "stream": true, "stream_options": { "include_usage": true }, "reasoning": { "enabled": false }, "thinking": { "type": "disabled" } }, "prompts": [ { "id": "http-caching", "text": "Explain HTTP caching to a developer in exactly five short numbered sentences. Mention Cache-Control, ETag, conditional requests, freshness, and private responses. Answer directly without a preamble." }, { "id": "json-transform", "text": "Convert these records into a JSON array sorted by score descending: Ada has 7, Ben has 3, Cy has 9. Each object must have name and score fields. Return only JSON, without Markdown." }, { "id": "python-function", "text": "Write a Python function that returns the median of a list of numbers, with ValueError for an empty list, without modifying the input. Include one even-length and one odd-length example. Keep the answer under 200 words." } ], "client_region": "Not independently verified", "percentile_definition": "Median is midpoint for even sample counts. p95 is nearest rank ceil(0.95*n), only when at least 20 valid observations exist. Failures and missing visible answers are counted separately, never assigned zero latency.", "timing_definition": "Client wall time from fetch invocation. first_token_ms is first nonempty content OR reasoning delta; first_content_ms excludes reasoning. Includes network, queueing and generation. Not provider-only latency.", "cost_definition": "Estimate using returned usage and publicly advertised catalog rates captured before run; not an invoice or observed charge. Cache counted only where usage reports it. No plan bonus applied.", "limitations": [ "Small sequential sample, one client/network and three short prompts; not a capacity, quality, long-context, or availability benchmark.", "No regional or competing-provider comparison. Empirical p95 is unstable in small samples, not a population estimate or SLA. Caches and routing may vary.", "Maximum 512 completion tokens per request; finish_reason length means truncation." ], "catalog": { "object": "list", "data": [ { "id": "deepseek/deepseek-v4.1-flash-1017", "object": "model", "created": 1793318400, "owned_by": "excession", "context_length": 1048576, "max_output_tokens": 131072, "pricing": { "currency": "USD", "input_per_million_tokens": "0.035", "cached_input_per_million_tokens": "0.005", "output_per_million_tokens": "0.50" } }, { "id": "deepseek-v4.1-flash-abliterated", "object": "model", "created": 1793318400, "owned_by": "excession", "context_length": 1048576, "max_output_tokens": 131072, "pricing": { "currency": "USD", "input_per_million_tokens": "0.20", "cached_input_per_million_tokens": "0.025", "output_per_million_tokens": "0.80" } } ] }, "summary": [ { "model": "deepseek/deepseek-v4.1-flash-1017", "requests": 20, "successful_streams": 0, "errors": 20, "visible_answers": 0, "truncated": 0, "no_visible_answer": 0, "median_first_token_ms": null, "median_first_content_ms": null, "p95_first_content_ms": null, "median_total_ms": null, "p95_total_ms": null, "median_completion_tokens": null, "usage_reported_requests": 0, "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "requests": 20, "successful_streams": 20, "errors": 0, "visible_answers": 20, "truncated": 3, "no_visible_answer": 0, "median_first_token_ms": 593.995, "median_first_content_ms": 3063.4449999999997, "p95_first_content_ms": 4495.19, "median_total_ms": 3563.2799999999997, "p95_total_ms": 4832.15, "median_completion_tokens": 331, "usage_reported_requests": 20, "estimated_cost_usd_at_catalog_rates": 0.004803400000000001 } ], "requests": [ { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 1, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:31.072Z", "http_status": 503, "headers_ms": 358.82, "first_token_ms": null, "first_content_ms": null, "total_ms": 361.61, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 1, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:31.434Z", "http_status": 200, "headers_ms": 1796.67, "first_token_ms": 1796.97, "first_content_ms": 3744.8, "total_ms": 3891.46, "completed": true, "finish_reason": "length", "usage": { "completion_tokens": 512, "completion_tokens_details": { "reasoning_tokens": 470 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 577, "cost": 0 }, "output": "1. HTTP caching stores reusable responses to reduce latency and network load.\n2. Cache-Control directives set caching rules and freshness lifetimes.\n3. An ETag is a validator that identifies a specific response version.\n", "reasoning_characters": 2252, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0004226 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 2, "prompt_id": "json-transform", "started_at": "2026-11-05T12:21:35.325Z", "http_status": 200, "headers_ms": 394.55, "first_token_ms": 394.64, "first_content_ms": 569.13, "total_ms": 681.4, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 82, "completion_tokens_details": { "reasoning_tokens": 54 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 156, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 185, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0000804 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 2, "prompt_id": "json-transform", "started_at": "2026-11-05T12:21:36.007Z", "http_status": 503, "headers_ms": 194.45, "first_token_ms": null, "first_content_ms": null, "total_ms": 194.58, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 3, "prompt_id": "python-function", "started_at": "2026-11-05T12:21:36.201Z", "http_status": 503, "headers_ms": 164.42, "first_token_ms": null, "first_content_ms": null, "total_ms": 164.54, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 3, "prompt_id": "python-function", "started_at": "2026-11-05T12:21:36.366Z", "http_status": 200, "headers_ms": 2227.95, "first_token_ms": 2228.07, "first_content_ms": 2880.87, "total_ms": 3454.41, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 345, "completion_tokens_details": { "reasoning_tokens": 210 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 419, "cost": 0 }, "output": "```python\ndef median(numbers):\n if not numbers:\n raise ValueError(\"median() arg is an empty list\")\n ordered = sorted(numbers) # copy; input is not modified\n n = len(ordered)\n mid = n // 2\n if n % 2:\n return ordered[mid]\n return (ordered[mid - 1] + ordered[mid]) / 2\n\n# Odd-length example\nprint(median([3, 1, 2])) # 2\n\n# Even-length example\nprint(median([4, 1, 3, 2])) # 2.5\n```", "reasoning_characters": 762, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0002908 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 4, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:39.820Z", "http_status": 200, "headers_ms": 384.5, "first_token_ms": 384.62, "first_content_ms": 3600.65, "total_ms": 4513.19, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 372, "completion_tokens_details": { "reasoning_tokens": 258 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 437, "cost": 0 }, "output": "1. HTTP caching stores responses so clients and intermediaries can reuse them instead of refetching.\n2. Cache-Control directives define who may cache a response and for how long, controlling freshness.\n3. An ETag is a validator that identifies a specific version of a response.\n4. Clients send conditional requests with validators like If-None-Match so the server can return 304 Not Modified when the cached copy is still valid.\n5. Mark user-specific responses private so shared caches don’t store them while browser caches can still reuse them.", "reasoning_characters": 1246, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0003106 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 4, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:44.334Z", "http_status": 503, "headers_ms": 141.43, "first_token_ms": null, "first_content_ms": null, "total_ms": 141.54, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 5, "prompt_id": "json-transform", "started_at": "2026-11-05T12:21:44.475Z", "http_status": 503, "headers_ms": 142.75, "first_token_ms": null, "first_content_ms": null, "total_ms": 142.86, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 5, "prompt_id": "json-transform", "started_at": "2026-11-05T12:21:44.618Z", "http_status": 200, "headers_ms": 598.29, "first_token_ms": 598.38, "first_content_ms": 709.04, "total_ms": 877.06, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 82, "completion_tokens_details": { "reasoning_tokens": 54 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 156, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 185, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0000804 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 6, "prompt_id": "python-function", "started_at": "2026-11-05T12:21:45.495Z", "http_status": 200, "headers_ms": 2671.77, "first_token_ms": 2671.89, "first_content_ms": 3391.63, "total_ms": 3882.29, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 399, "completion_tokens_details": { "reasoning_tokens": 232 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 473, "cost": 0 }, "output": "```python\ndef median(nums):\n if not nums:\n raise ValueError(\"median() arg is an empty list\")\n s = sorted(nums) # copy; input is not modified\n n = len(s)\n mid = n // 2\n if n % 2:\n return s[mid]\n return (s[mid - 1] + s[mid]) / 2\n\n# Odd-length example\nprint(median([3, 1, 2])) # 2\n\n# Even-length example\nprint(median([4, 1, 3, 2])) # 2.5\n\n# Input remains unchanged\nnums = [3, 1, 2]\nmedian(nums)\nprint(nums) # [3, 1, 2]\n```", "reasoning_characters": 845, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00033400000000000004 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 6, "prompt_id": "python-function", "started_at": "2026-11-05T12:21:49.378Z", "http_status": 503, "headers_ms": 160.17, "first_token_ms": null, "first_content_ms": null, "total_ms": 160.29, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 7, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:49.538Z", "http_status": 503, "headers_ms": 175.88, "first_token_ms": null, "first_content_ms": null, "total_ms": 176, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 7, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:49.714Z", "http_status": 200, "headers_ms": 376.06, "first_token_ms": 376.16, "first_content_ms": 3811.15, "total_ms": 4154.07, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 317, "completion_tokens_details": { "reasoning_tokens": 215 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 382, "cost": 0 }, "output": "1. HTTP caching stores responses so clients and intermediaries can reuse them instead of refetching.\n2. Cache-Control directives define freshness lifetime and whether a response is cacheable by shared or private caches.\n3. Freshness determines whether a cached response can be served without contacting the origin.\n4. ETag identifies a specific response version, enabling conditional requests like If-None-Match to validate stale entries.\n5. Mark user-specific data as private responses so shared caches do not store it.", "reasoning_characters": 1118, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00026660000000000003 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 8, "prompt_id": "json-transform", "started_at": "2026-11-05T12:21:53.868Z", "http_status": 200, "headers_ms": 1007.16, "first_token_ms": 1007.3, "first_content_ms": 1149.89, "total_ms": 1271.08, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 84, "completion_tokens_details": { "reasoning_tokens": 56 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 158, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 196, "error": null, "estimated_cost_usd_at_catalog_rates": 0.000082 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 8, "prompt_id": "json-transform", "started_at": "2026-11-05T12:21:55.139Z", "http_status": 503, "headers_ms": 161.6, "first_token_ms": null, "first_content_ms": null, "total_ms": 161.71, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 9, "prompt_id": "python-function", "started_at": "2026-11-05T12:21:55.301Z", "http_status": 503, "headers_ms": 137.45, "first_token_ms": null, "first_content_ms": null, "total_ms": 137.59, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 9, "prompt_id": "python-function", "started_at": "2026-11-05T12:21:55.438Z", "http_status": 200, "headers_ms": 2179.46, "first_token_ms": 2179.55, "first_content_ms": 3072.48, "total_ms": 3542.84, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 368, "completion_tokens_details": { "reasoning_tokens": 236 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 442, "cost": 0 }, "output": "```python\ndef median(nums):\n if not nums:\n raise ValueError(\"median() arg is an empty list\")\n s = sorted(nums) # copy, input unchanged\n n = len(s)\n mid = n // 2\n if n % 2:\n return s[mid]\n return (s[mid - 1] + s[mid]) / 2\n\n# Odd-length example\nprint(median([3, 1, 2])) # 2\n\n# Even-length example\nprint(median([4, 1, 3, 2])) # 2.5\n```", "reasoning_characters": 916, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00030920000000000003 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 10, "prompt_id": "http-caching", "started_at": "2026-11-05T12:21:58.981Z", "http_status": 200, "headers_ms": 450.14, "first_token_ms": 450.23, "first_content_ms": 3677.63, "total_ms": 4079.47, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 369, "completion_tokens_details": { "reasoning_tokens": 261 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 434, "cost": 0 }, "output": "1. HTTP caching stores responses so clients and intermediaries can reuse them without contacting the origin every time.\n2. Cache-Control directives define freshness lifetime and whether a response is public, private, or must revalidate.\n3. ETag gives a response a validator so clients can make conditional requests with If-None-Match.\n4. Conditional requests let caches and clients revalidate stale entries and receive 304 Not Modified when unchanged.\n5. Private responses use Cache-Control: private so shared caches do not store user-specific data.", "reasoning_characters": 1256, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0003082 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 10, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:03.061Z", "http_status": 503, "headers_ms": 1083.51, "first_token_ms": null, "first_content_ms": null, "total_ms": 1085.41, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 11, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:04.147Z", "http_status": 503, "headers_ms": 168.43, "first_token_ms": null, "first_content_ms": null, "total_ms": 169.49, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 11, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:04.316Z", "http_status": 200, "headers_ms": 400.22, "first_token_ms": 400.56, "first_content_ms": 594.14, "total_ms": 705.3, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 82, "completion_tokens_details": { "reasoning_tokens": 54 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 156, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 185, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0000804 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 12, "prompt_id": "python-function", "started_at": "2026-11-05T12:22:05.022Z", "http_status": 200, "headers_ms": 2434.65, "first_token_ms": 2434.77, "first_content_ms": 3054.41, "total_ms": 3583.72, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 304, "completion_tokens_details": { "reasoning_tokens": 171 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 378, "cost": 0 }, "output": "```python\ndef median(numbers):\n if not numbers:\n raise ValueError(\"median() arg is an empty list\")\n ordered = sorted(numbers) # copy; input unchanged\n n = len(ordered)\n mid = n // 2\n if n % 2:\n return ordered[mid]\n return (ordered[mid - 1] + ordered[mid]) / 2\n\n# Odd-length example\nprint(median([7, 1, 3])) # 3\n\n# Even-length example\nprint(median([4, 1, 3, 2])) # 2.5\n```", "reasoning_characters": 724, "error": null, "estimated_cost_usd_at_catalog_rates": 0.000258 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 12, "prompt_id": "python-function", "started_at": "2026-11-05T12:22:08.605Z", "http_status": 503, "headers_ms": 164.54, "first_token_ms": null, "first_content_ms": null, "total_ms": 164.68, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 13, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:08.770Z", "http_status": 503, "headers_ms": 135.73, "first_token_ms": null, "first_content_ms": null, "total_ms": 135.89, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 13, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:08.906Z", "http_status": 200, "headers_ms": 375.83, "first_token_ms": 375.92, "first_content_ms": 5028.63, "total_ms": 5180.41, "completed": true, "finish_reason": "length", "usage": { "completion_tokens": 512, "completion_tokens_details": { "reasoning_tokens": 502 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 577, "cost": 0 }, "output": "1. HTTP caching stores reusable responses to cut", "reasoning_characters": 2411, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0004226 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 14, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:14.087Z", "http_status": 200, "headers_ms": 589.28, "first_token_ms": 589.61, "first_content_ms": 744.82, "total_ms": 881.51, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 83, "completion_tokens_details": { "reasoning_tokens": 55 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 157, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 189, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00008120000000000001 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 14, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:14.969Z", "http_status": 503, "headers_ms": 189.55, "first_token_ms": null, "first_content_ms": null, "total_ms": 189.75, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 15, "prompt_id": "python-function", "started_at": "2026-11-05T12:22:15.159Z", "http_status": 503, "headers_ms": 154.15, "first_token_ms": null, "first_content_ms": null, "total_ms": 154.25, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 15, "prompt_id": "python-function", "started_at": "2026-11-05T12:22:15.313Z", "http_status": 200, "headers_ms": 1640.01, "first_token_ms": 1640.09, "first_content_ms": 2617.4, "total_ms": 3055.57, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 369, "completion_tokens_details": { "reasoning_tokens": 235 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 443, "cost": 0 }, "output": "```python\ndef median(numbers):\n if not numbers:\n raise ValueError(\"median() arg is an empty list\")\n ordered = sorted(numbers) # new list; input unchanged\n n = len(ordered)\n mid = n // 2\n if n % 2:\n return ordered[mid]\n return (ordered[mid - 1] + ordered[mid]) / 2\n\n# Odd-length example\nprint(median([3, 1, 2])) # 2\n\n# Even-length example\nprint(median([4, 1, 3, 2])) # 2.5\n```", "reasoning_characters": 910, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00031 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 16, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:18.369Z", "http_status": 200, "headers_ms": 358.94, "first_token_ms": 359.05, "first_content_ms": 4223.37, "total_ms": 4581.49, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 390, "completion_tokens_details": { "reasoning_tokens": 287 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 455, "cost": 0 }, "output": "1. HTTP caching stores responses so clients and intermediaries can reuse them instead of refetching.\n2. Cache-Control directives define freshness lifetime and whether a response is public or private.\n3. ETag identifies a specific response version so caches and clients can validate it.\n4. Conditional requests use validators like If-None-Match to ask the server whether the cached response is still current.\n5. Mark private responses with Cache-Control: private so shared caches don’t store user-specific data.", "reasoning_characters": 1356, "error": null, "estimated_cost_usd_at_catalog_rates": 0.000325 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 16, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:22.950Z", "http_status": 503, "headers_ms": 194.69, "first_token_ms": null, "first_content_ms": null, "total_ms": 194.82, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 17, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:23.145Z", "http_status": 503, "headers_ms": 152.18, "first_token_ms": null, "first_content_ms": null, "total_ms": 152.27, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 17, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:23.297Z", "http_status": 200, "headers_ms": 614.98, "first_token_ms": 615.24, "first_content_ms": 732.91, "total_ms": 843.67, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 76, "completion_tokens_details": { "reasoning_tokens": 48 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 150, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 153, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00007560000000000001 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 18, "prompt_id": "python-function", "started_at": "2026-11-05T12:22:24.141Z", "http_status": 200, "headers_ms": 687.96, "first_token_ms": 688.04, "first_content_ms": 3515.19, "total_ms": 3971.6, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 309, "completion_tokens_details": { "reasoning_tokens": 171 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 383, "cost": 0 }, "output": "```python\ndef median(nums):\n if not nums:\n raise ValueError(\"median() arg is an empty list\")\n ordered = sorted(nums) # copy; input is not modified\n n = len(ordered)\n mid = n // 2\n if n % 2 == 1:\n return ordered[mid]\n return (ordered[mid - 1] + ordered[mid]) / 2\n\n# Odd-length example\nprint(median([3, 1, 2])) # 2\n\n# Even-length example\nprint(median([4, 1, 3, 2])) # 2.5\n```", "reasoning_characters": 621, "error": null, "estimated_cost_usd_at_catalog_rates": 0.000262 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 18, "prompt_id": "python-function", "started_at": "2026-11-05T12:22:28.113Z", "http_status": 503, "headers_ms": 166.25, "first_token_ms": null, "first_content_ms": null, "total_ms": 166.51, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 19, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:28.279Z", "http_status": 503, "headers_ms": 146.03, "first_token_ms": null, "first_content_ms": null, "total_ms": 146.24, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 19, "prompt_id": "http-caching", "started_at": "2026-11-05T12:22:28.425Z", "http_status": 200, "headers_ms": 371.29, "first_token_ms": 371.49, "first_content_ms": 4495.19, "total_ms": 4832.15, "completed": true, "finish_reason": "length", "usage": { "completion_tokens": 512, "completion_tokens_details": { "reasoning_tokens": 436 }, "prompt_tokens": 65, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 577, "cost": 0 }, "output": "1. HTTP caching stores reusable responses to reduce latency and network load.\n2. Cache-Control sets freshness lifetime and controls whether shared caches may store a response.\n3. An ETag is a validator that identifies a specific version of a cached response.\n4. Conditional requests like If-None-Match let the server return 304 Not Modified when the ETag still matches.\n", "reasoning_characters": 2142, "error": null, "estimated_cost_usd_at_catalog_rates": 0.0004226 }, { "model": "deepseek-v4.1-flash-abliterated", "sample": 20, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:33.258Z", "http_status": 200, "headers_ms": 402.14, "first_token_ms": 402.38, "first_content_ms": 576.95, "total_ms": 814.64, "completed": true, "finish_reason": "stop", "usage": { "completion_tokens": 83, "completion_tokens_details": { "reasoning_tokens": 55 }, "prompt_tokens": 74, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 157, "cost": 0 }, "output": "[{\"name\":\"Cy\",\"score\":9},{\"name\":\"Ada\",\"score\":7},{\"name\":\"Ben\",\"score\":3}]", "reasoning_characters": 189, "error": null, "estimated_cost_usd_at_catalog_rates": 0.00008120000000000001 }, { "model": "deepseek/deepseek-v4.1-flash-1017", "sample": 20, "prompt_id": "json-transform", "started_at": "2026-11-05T12:22:34.072Z", "http_status": 503, "headers_ms": 129.43, "first_token_ms": null, "first_content_ms": null, "total_ms": 129.55, "completed": false, "finish_reason": null, "usage": null, "output": "", "reasoning_characters": 0, "error": "HTTP 503", "estimated_cost_usd_at_catalog_rates": null } ] }