{
  "audited_at": "2026-11-11T04:37:06.932899+00:00",
  "scope": "Regrade of retained synthetic outputs from October 15\u201316, 2026. No new model inference or speed benchmark.",
  "grader_sha256": "56e0f1beaa2fa8ea268e9ca62f08cb51ca301adfef2b4636e0c8ef0f7499a3bd",
  "published_summary_sha256": "8e5ee7567ceac83bf7499a2bb308c310b00a4ce556d18e8fac5e7be1a6ce59db",
  "agent_runs": [
    {
      "source_file": "results/goose-opus_20261016T095454Z.json",
      "original_result_sha256": "b5ca25cf2c7b0b74ade6e5e62b3ff75ee4865584eacea591bb53533cdc29ea0c",
      "version": "1.54.0",
      "started_utc": "2026-10-16T09:54:55Z",
      "finished_utc": "2026-10-16T09:57:04Z",
      "tool": "goose",
      "model": "claude-opus-5-5",
      "recorded_passed": 9,
      "recorded_total": 9,
      "trials": [
        {
          "task": "t1_create_file",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 9.1,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 13.1,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55, the order service failed about one in five requests, and about 1,100 customers saw slow checkouts or errors for 41 minutes. The root cause was a bug in the nightly report worker, deployed at 01:55: when a slow export step returned early, the worker's new long-running transaction was left open, which leaked database connections until all 200 slots were used up by sessions sitting idle in transaction. The team stopped the outage by pausing the report queue and killing the stuck sessions, and the fixes are a rollback in a finally block, a five-minute idle-in-transaction timeout, a separate 20-connection pool for the report worker, and a new alert on idle-in-transaction sessions.\n",
          "output_sha256": "4c318dd8137cbf30357a1acc308f57bea41aa061781c24f5ee66aa80f46d804b",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 11.6,
          "output_file": "count.txt",
          "output": "37",
          "output_sha256": "7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25"
        },
        {
          "task": "t1_create_file",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 8.1,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 10.3,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55, the order service failed about one request in five because all 200 database connection slots were used up, which caused slow checkouts or errors for about 1,100 customers over 41 minutes. The root cause was a bug in the nightly report worker, deployed at 01:55: when a slow export step failed, the code returned early without closing its long transaction, so each failed report held a database connection idle in transaction for up to an hour. Errors stopped after the report queue was paused and the stuck sessions were killed, and the fixes are to always roll back in a finally block, set a five-minute idle-in-transaction timeout, give the report worker its own pool of 20 connections, and add an alert on idle-in-transaction sessions.\n",
          "output_sha256": "b3d858880b5e7ac5e78a94e2253ff29412fb9b8980ab57a352c8658eaacef532",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 13.1,
          "output_file": "count.txt",
          "output": "37",
          "output_sha256": "7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25"
        },
        {
          "task": "t1_create_file",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 6.4,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 9.2,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55, the order service failed about one request in five, and about 1,100 customers saw slow checkouts or errors. The root cause was a bug in the nightly report worker, deployed at 01:55: when a slow export step failed, the code returned early without closing its long transaction, so connections leaked until all 200 database connection slots were used. The team paused the report queue and killed the stuck sessions, and the fixes are to always roll back in a finally block, add a five-minute idle-in-transaction timeout, give the report worker its own 20-connection pool, and alert on idle-in-transaction sessions.\n",
          "output_sha256": "91a38da014230ac355dd8bd4702d66c9f76ddf5f76dab3f05f022e1e73ba169f",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 9.0,
          "output_file": "count.txt",
          "output": "37\n",
          "output_sha256": "b58a3da5fde2680191877ec88a1aa7d06927cc3b30cdf0d0db8c39b488891576"
        }
      ]
    },
    {
      "source_file": "results/goose_20261015T070530Z.json",
      "original_result_sha256": "46a7eda437fbfdf004000f965218107bf60dd248fd9d150dcd67da7bc19c975f",
      "version": "1.54.0",
      "started_utc": "2026-10-15T07:05:30Z",
      "finished_utc": "2026-10-15T07:09:21Z",
      "tool": "goose",
      "model": "qwen2.5:3b",
      "recorded_passed": 5,
      "recorded_total": 9,
      "trials": [
        {
          "task": "t1_create_file",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 49.5,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 35.3,
          "output_file": "summary.txt",
          "output": "The root cause of the issue was a leaky transaction, where long-running database tasks such as nightly report generation kept database connections open unnecessarily. A solution involved adding a rollback in the finally block, ensuring a timeout for idle transactions, and utilizing a separate connection pool for the report worker.",
          "output_sha256": "5a8b41d537c2b7a12d3e4cf973178e024855ab4be6938934224f1a82fe6e1e40",
          "sentence_count": 2,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 1,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 15.1,
          "output_file": "count.txt",
          "output": "38 data.csv\n",
          "output_sha256": "10627844c4242d103dd0e55b0479f5cef5420fcde05e9623654fd97d1e9741f4"
        },
        {
          "task": "t1_create_file",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 9.0,
          "output_file": "hello.txt",
          "output": "ADF bench OK\n",
          "output_sha256": "1c10a449b38f2d0e48f9c88275e740e0d15709eaebfa1a73b222c065cd6877af"
        },
        {
          "task": "t2_read_summarize",
          "rep": 2,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 33.1,
          "output_file": "summary.txt",
          "output": null,
          "output_sha256": null,
          "sentence_count": 0,
          "within_requested_three_sentences": false
        },
        {
          "task": "t3_shell_count",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 12.4,
          "output_file": "count.txt",
          "output": "37\n",
          "output_sha256": "b58a3da5fde2680191877ec88a1aa7d06927cc3b30cdf0d0db8c39b488891576"
        },
        {
          "task": "t1_create_file",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 10.5,
          "output_file": "hello.txt",
          "output": "ADF bench OK\n",
          "output_sha256": "1c10a449b38f2d0e48f9c88275e740e0d15709eaebfa1a73b222c065cd6877af"
        },
        {
          "task": "t2_read_summarize",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 19.8,
          "output_file": "summary.txt",
          "output": "To be provided",
          "output_sha256": "5275e2c4a45e3fb69db39e4801ca316eb894a40f1c86675d7737e4a040beaa91",
          "sentence_count": 1,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 21.6,
          "output_file": "count.txt",
          "output": "38 data.csv\n",
          "output_sha256": "10627844c4242d103dd0e55b0479f5cef5420fcde05e9623654fd97d1e9741f4"
        }
      ]
    },
    {
      "source_file": "results/hermes-opus_20261016T100805Z.json",
      "original_result_sha256": "3988211894f60f8c1477d9cd67f1dbcfce373a540d36a05bb3e99f3a8558ae5f",
      "version": "Hermes Agent v0.22.1+72.g450ad50 (2026.9.24), commit 450ad5055eb3c4852d96676ee99287d618bd283a",
      "started_utc": "2026-10-16T10:08:06Z",
      "finished_utc": "2026-10-16T10:11:56Z",
      "tool": "hermes",
      "model": "claude-opus-5-5",
      "recorded_passed": 9,
      "recorded_total": 9,
      "trials": [
        {
          "task": "t1_create_file",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 17.7,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 24.8,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55, the order service failed about one request in five because all 200 database connection slots were used up, which caused slow checkouts or errors for about 1,100 customers over 41 minutes.\nThe root cause was a bug in the nightly report worker, deployed at 01:55: when a slow export step failed, the code returned early without closing the report's long transaction, so each failed report kept a database connection open, idle in transaction, for up to an hour.\nThe incident was stopped by pausing the report queue and killing the stuck sessions, and the fixes are to always roll back in a finally block, add a five-minute idle-in-transaction timeout for the report role, give the report worker its own pool of 20 connections, and add an alert on idle-in-transaction sessions.\n",
          "output_sha256": "e6048a4027e06fd6bcbec7a5c0ba9ceffd58f6e52b9468adaece6f72ae737d45",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 25.2,
          "output_file": "count.txt",
          "output": "37\n",
          "output_sha256": "b58a3da5fde2680191877ec88a1aa7d06927cc3b30cdf0d0db8c39b488891576"
        },
        {
          "task": "t1_create_file",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 22.1,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 25.7,
          "output_file": "summary.txt",
          "output": "From about 02:14 to 02:55, the order service failed about one request in five, and roughly 1,100 customers saw slow checkouts or errors because all 200 database connections were in use. The root cause was a bug in the nightly report worker, deployed at 01:55: on a slow export step it returned early without closing its long-running transaction, so each failed report kept a database connection open for up to an hour. The incident was resolved by pausing the report queue and killing the stuck sessions, and the follow-up fixes are always rolling back in a finally block, a five-minute idle-in-transaction timeout, a separate 20-connection pool for the report worker, and a new alert on idle-in-transaction sessions.\n",
          "output_sha256": "670847301db65adbdd6695f8e3f15ccf597ca701d2e55001448fbaac4698dc59",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 22.3,
          "output_file": "count.txt",
          "output": "37\n",
          "output_sha256": "b58a3da5fde2680191877ec88a1aa7d06927cc3b30cdf0d0db8c39b488891576"
        },
        {
          "task": "t1_create_file",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 18.8,
          "output_file": "hello.txt",
          "output": "ADF bench OK\n",
          "output_sha256": "1c10a449b38f2d0e48f9c88275e740e0d15709eaebfa1a73b222c065cd6877af"
        },
        {
          "task": "t2_read_summarize",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 18.3,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55, the order service failed about one request in five, and about 1,100 customers saw slow checkouts or errors over 41 minutes. The root cause was a bug in the nightly report worker, deployed at 01:55: when a slow export step failed, the code returned early without closing its long transaction, so each failed report held a database connection for up to an hour until all 200 slots were used up. The team paused the report queue and killed the stuck sessions, and the follow-up fixes are a rollback in a finally block, a five-minute idle-in-transaction timeout, a separate 20-connection pool for the report worker, and a new alert on idle-in-transaction sessions.\n",
          "output_sha256": "13b054d55835613cfe3fd67876a015a8bf54caaf5ff869d81894bb31b4d133eb",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 21.2,
          "output_file": "count.txt",
          "output": "37\n",
          "output_sha256": "b58a3da5fde2680191877ec88a1aa7d06927cc3b30cdf0d0db8c39b488891576"
        }
      ]
    },
    {
      "source_file": "results/hermes_20261015T070929Z.json",
      "original_result_sha256": "dbb158ec8f3739780f515ccd035786727250511ea57df77b80cf0ea1892d0cba",
      "version": "Hermes Agent v0.22.1+72.g450ad50 (2026.9.24), commit 450ad5055eb3c4852d96676ee99287d618bd283a",
      "started_utc": "2026-10-15T07:09:29Z",
      "finished_utc": "2026-10-15T07:44:36Z",
      "tool": "hermes",
      "model": "qwen2.5:3b",
      "recorded_passed": 0,
      "recorded_total": 9,
      "trials": [
        {
          "task": "t1_create_file",
          "rep": 1,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 467.5,
          "output_file": "hello.txt",
          "output": null,
          "output_sha256": null
        },
        {
          "task": "t2_read_summarize",
          "rep": 1,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 156.1,
          "output_file": "summary.txt",
          "output": null,
          "output_sha256": null,
          "sentence_count": 0,
          "within_requested_three_sentences": false
        },
        {
          "task": "t3_shell_count",
          "rep": 1,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 74.5,
          "output_file": "count.txt",
          "output": null,
          "output_sha256": null
        },
        {
          "task": "t1_create_file",
          "rep": 2,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 98.2,
          "output_file": "hello.txt",
          "output": null,
          "output_sha256": null
        },
        {
          "task": "t2_read_summarize",
          "rep": 2,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 84.5,
          "output_file": "summary.txt",
          "output": null,
          "output_sha256": null,
          "sentence_count": 0,
          "within_requested_three_sentences": false
        },
        {
          "task": "t3_shell_count",
          "rep": 2,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 151.1,
          "output_file": "count.txt",
          "output": null,
          "output_sha256": null
        },
        {
          "task": "t1_create_file",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 177.4,
          "output_file": "hello.txt",
          "output": null,
          "output_sha256": null
        },
        {
          "task": "t2_read_summarize",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 405.4,
          "output_file": "summary.txt",
          "output": null,
          "output_sha256": null,
          "sentence_count": 0,
          "within_requested_three_sentences": false
        },
        {
          "task": "t3_shell_count",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 100.0,
          "output_file": "count.txt",
          "output": null,
          "output_sha256": null
        }
      ]
    },
    {
      "source_file": "results/opencode-opus_20261016T100140Z.json",
      "original_result_sha256": "94974d380d05de6eed62232260380cbbdf79fe6d9adbdb2ab3e32ce42a808cf5",
      "version": "1.19.2",
      "started_utc": "2026-10-16T10:01:40Z",
      "finished_utc": "2026-10-16T10:04:03Z",
      "tool": "opencode",
      "model": "claude-opus-5-5",
      "recorded_passed": 9,
      "recorded_total": 9,
      "trials": [
        {
          "task": "t1_create_file",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 11.2,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 16.8,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55, the order service failed about one request in five, and roughly 1,100 customers saw slow checkouts or errors. The root cause was a bug in the nightly report worker, deployed at 01:55: it returned early on a slow export step without closing its long-running transaction, so each failed report leaked a database connection until all 200 connection slots were used up. The incident was mitigated by pausing the report queue and killing the stuck sessions, and the follow-up fixes are a rollback in a finally block, a five-minute idle-in-transaction timeout, a separate 20-connection pool for the report worker, and a new alert on idle-in-transaction sessions.\n",
          "output_sha256": "b138a79ce3a1b6049e2c35aea7a57976d62b6a5709287c7e6bedfb39a5adb791",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 10.1,
          "output_file": "count.txt",
          "output": "37\n",
          "output_sha256": "b58a3da5fde2680191877ec88a1aa7d06927cc3b30cdf0d0db8c39b488891576"
        },
        {
          "task": "t1_create_file",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 10.0,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 16.3,
          "output_file": "summary.txt",
          "output": "From 02:14 to 02:55 the order service failed about one request in five, causing slow checkouts or errors for roughly 1,100 customers over 41 minutes. The root cause was a bug in the nightly report worker, deployed at 01:55, which returned early on a slow export step without closing its long-running transaction, leaking database connections until all 200 slots were exhausted. The incident was resolved by pausing the report queue and killing stuck sessions, and follow-up fixes include rollback in a finally block, a five-minute idle-in-transaction timeout, a separate 20-connection pool for the report worker, and a new idle-in-transaction alert.\n",
          "output_sha256": "29bc32b1a3a50fe5fb906f04bd80bfef984fca487ca48175221ccfdb493efddc",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 12.2,
          "output_file": "count.txt",
          "output": "37",
          "output_sha256": "7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25"
        },
        {
          "task": "t1_create_file",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 11.0,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 15.3,
          "output_file": "summary.txt",
          "output": "From about 02:14 to 02:55, the order service failed roughly one in five requests, and about 1,100 customers saw slow checkouts or errors over 41 minutes. The root cause was a change to the nightly report worker, deployed at 01:55, that wrapped each report in one long transaction; a bug returned early on a slow export step without closing that transaction, leaking connections until all 200 database connection slots were used up. The incident was resolved by pausing the report queue and killing the stuck sessions, and the follow-up fixes are rollback in a finally block, a five-minute idle-in-transaction timeout, a separate 20-connection pool for the report worker, and a new alert on idle-in-transaction sessions.\n",
          "output_sha256": "dcc0923209587df4ff8bfa598b55480c9173808bf5e3c6b972d40dbd3d93a99e",
          "sentence_count": 3,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 10.9,
          "output_file": "count.txt",
          "output": "37",
          "output_sha256": "7a61b53701befdae0eeeffaecc73f14e20b537bb0f8b91ad7c2936dc63562b25"
        }
      ]
    },
    {
      "source_file": "results/opencode_20261016T093354Z.json",
      "original_result_sha256": "47e10683ab8c87be95ec3cda1d59418dfd724a7b0c36789550103d729907fc96",
      "version": "1.19.2",
      "started_utc": "2026-10-16T09:33:55Z",
      "finished_utc": "2026-10-16T09:44:16Z",
      "tool": "opencode",
      "model": "qwen2.5:3b",
      "recorded_passed": 3,
      "recorded_total": 9,
      "trials": [
        {
          "task": "t1_create_file",
          "rep": 1,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 193.5,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 1,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 42.8,
          "output_file": "summary.txt",
          "output": "### Summary\nAs per the root cause found, the issue was directly due to a misconfigured security policy.",
          "output_sha256": "598c597dd6e829cd56796dc622e80286e821d73e2ca8cf4f66142e2bee7c919c",
          "sentence_count": 1,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 1,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 88.0,
          "output_file": "count.txt",
          "output": "18",
          "output_sha256": "4ec9599fc203d176a301536c2e091a19bc852759b255bd6818810a42c5fed14a"
        },
        {
          "task": "t1_create_file",
          "rep": 2,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 20.8,
          "output_file": "hello.txt",
          "output": "ADF bench OK\n",
          "output_sha256": "1c10a449b38f2d0e48f9c88275e740e0d15709eaebfa1a73b222c065cd6877af"
        },
        {
          "task": "t2_read_summarize",
          "rep": 2,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 37.7,
          "output_file": "summary.txt",
          "output": "The root cause was identified as a missing dependency in the project.\\n",
          "output_sha256": "84f31a720e2f4f47c9612b4760f75e06c68719e6a3145f613aa0767b9d389038",
          "sentence_count": 1,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 2,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 26.2,
          "output_file": "count.txt",
          "output": "38\n",
          "output_sha256": "a2c6c14110a317833ac8f2fbae7080ee844af7469b2aa702876548dcf4c58077"
        },
        {
          "task": "t1_create_file",
          "rep": 3,
          "recorded_pass": true,
          "replayed_pass": true,
          "wall_s": 18.5,
          "output_file": "hello.txt",
          "output": "ADF bench OK",
          "output_sha256": "8b66adf875edaf479394c9428e2116b2503f56103176d389487792342e88e041"
        },
        {
          "task": "t2_read_summarize",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 34.5,
          "output_file": "summary.txt",
          "output": "The root cause was [read content of incident.txt].",
          "output_sha256": "dc1385616e18a0865a534f9b5cf97f438e187e4f0eaffc5abf0f634d7a2a47a2",
          "sentence_count": 1,
          "within_requested_three_sentences": true
        },
        {
          "task": "t3_shell_count",
          "rep": 3,
          "recorded_pass": false,
          "replayed_pass": false,
          "wall_s": 24.0,
          "output_file": "count.txt",
          "output": "38\n",
          "output_sha256": "a2c6c14110a317833ac8f2fbae7080ee844af7469b2aa702876548dcf4c58077"
        }
      ]
    }
  ],
  "runtime_runs": [
    {
      "source_file": "results/llamacpp_20261016T083349Z.json",
      "original_result_sha256": "10db4434962574dd9bbf13ad65b25f3e7721c13cf4b3abf2dd0b7da6c52e9853",
      "version": "version: 0.7.0-dev (build 11681, commit 5e03a1ff4)",
      "started_utc": "2026-10-16T08:33:49Z",
      "finished_utc": "2026-10-16T08:42:44Z",
      "tool": "llamacpp",
      "first_request": {
        "runs": [
          {
            "mode": "cold",
            "i": 1,
            "start_to_first_answer_s": 77.002358549,
            "r": {
              "server_ready_s": 72.759,
              "first_request_wall_s": 0.779
            }
          },
          {
            "mode": "cold",
            "i": 2,
            "start_to_first_answer_s": 11.826173984,
            "r": {
              "server_ready_s": 7.923,
              "first_request_wall_s": 0.794
            }
          },
          {
            "mode": "cold",
            "i": 3,
            "start_to_first_answer_s": 7.688101869,
            "r": {
              "server_ready_s": 3.699,
              "first_request_wall_s": 0.728
            }
          },
          {
            "mode": "warmcache",
            "i": 1,
            "start_to_first_answer_s": 6.180338056,
            "r": {
              "server_ready_s": 2.261,
              "first_request_wall_s": 0.66
            }
          },
          {
            "mode": "warmcache",
            "i": 2,
            "start_to_first_answer_s": 6.96801876,
            "r": {
              "server_ready_s": 2.929,
              "first_request_wall_s": 0.766
            }
          },
          {
            "mode": "warmcache",
            "i": 3,
            "start_to_first_answer_s": 6.081397044,
            "r": {
              "server_ready_s": 2.533,
              "first_request_wall_s": 0.708
            }
          }
        ],
        "cold_median_s": 11.83,
        "warmcache_median_s": 6.18,
        "note": "container start -> server ready -> first 1-token answer; cold = GGUF evicted from page cache"
      },
      "model": {
        "gguf": "same file as Ollama qwen2.5:3b",
        "gguf_blob": "sha256:5ee4f07cdb9beadbbb293e85803c569b01bd37ed059d2715faa7bb405f31caa6"
      }
    },
    {
      "source_file": "results/llamacpp_20261016T104105Z.json",
      "original_result_sha256": "19e0c330ccf80bba73f400409b037be6d98abefe724949a03088c479ab86d669",
      "version": "version: 0.7.0-dev (build 11681, commit 5e03a1ff4)",
      "started_utc": "2026-10-16T10:44:06Z",
      "finished_utc": "2026-10-16T10:51:14Z",
      "tool": "llamacpp",
      "first_request": {
        "runs": [
          {
            "mode": "cold",
            "i": 1,
            "start_to_first_answer_s": 9.431246273,
            "r": {
              "server_ready_s": 6.957,
              "first_request_wall_s": 0.686
            }
          },
          {
            "mode": "cold",
            "i": 2,
            "start_to_first_answer_s": 6.076635476,
            "r": {
              "server_ready_s": 2.878,
              "first_request_wall_s": 0.637
            }
          },
          {
            "mode": "cold",
            "i": 3,
            "start_to_first_answer_s": 6.598380043,
            "r": {
              "server_ready_s": 2.727,
              "first_request_wall_s": 0.77
            }
          },
          {
            "mode": "warmcache",
            "i": 1,
            "start_to_first_answer_s": 6.521287613,
            "r": {
              "server_ready_s": 3.149,
              "first_request_wall_s": 0.669
            }
          },
          {
            "mode": "warmcache",
            "i": 2,
            "start_to_first_answer_s": 6.216916393,
            "r": {
              "server_ready_s": 3.125,
              "first_request_wall_s": 0.68
            }
          },
          {
            "mode": "warmcache",
            "i": 3,
            "start_to_first_answer_s": 5.746099047,
            "r": {
              "server_ready_s": 2.663,
              "first_request_wall_s": 0.633
            }
          }
        ],
        "cold_median_s": 6.6,
        "warmcache_median_s": 6.22,
        "note": "container start -> server ready -> first 1-token answer; cold = GGUF evicted from page cache"
      },
      "model": {
        "gguf": "same file as Ollama qwen2.5:3b",
        "gguf_blob": "sha256:5ee4f07cdb9beadbbb293e85803c569b01bd37ed059d2715faa7bb405f31caa6"
      }
    },
    {
      "source_file": "results/lmstudio_20261015T080148Z.json",
      "original_result_sha256": "6852410b07b14ebe4d1ada558cd40319ff1c7fa95637c41797ce46e9c87ee219",
      "version": "llmster 0.0.25-1",
      "started_utc": "2026-10-15T08:01:48Z",
      "finished_utc": "2026-10-15T08:11:43Z",
      "tool": "lmstudio",
      "first_request": {},
      "model": {
        "gguf": "same file as Ollama qwen2.5:3b",
        "gguf_blob": "sha256:5ee4f07cdb9beadbbb293e85803c569b01bd37ed059d2715faa7bb405f31caa6",
        "import": "lms import -l (symlink)"
      }
    },
    {
      "source_file": "results/localai_20261016T103402Z.json",
      "original_result_sha256": "e03ca0cd4fc4ddf038dba3c4bc68a0b1a845360efcb3f3baad295e2352dc4a95",
      "version": "Version: v4.11.0",
      "started_utc": "2026-10-16T10:34:02Z",
      "finished_utc": "2026-10-16T10:41:05Z",
      "tool": "localai",
      "first_request": {
        "runs": [
          {
            "mode": "cold",
            "i": 1,
            "start_to_first_answer_s": 10.132927809,
            "r": {
              "server_ready_s": 0.075,
              "first_request_wall_s": 8.281
            }
          },
          {
            "mode": "cold",
            "i": 2,
            "start_to_first_answer_s": 8.579507736,
            "r": {
              "server_ready_s": 0.027,
              "first_request_wall_s": 6.347
            }
          },
          {
            "mode": "cold",
            "i": 3,
            "start_to_first_answer_s": 8.710187689,
            "r": {
              "server_ready_s": 0.026,
              "first_request_wall_s": 6.35
            }
          },
          {
            "mode": "warmcache",
            "i": 1,
            "start_to_first_answer_s": 8.626740764,
            "r": {
              "server_ready_s": 0.023,
              "first_request_wall_s": 6.395
            }
          },
          {
            "mode": "warmcache",
            "i": 2,
            "start_to_first_answer_s": 8.802462706,
            "r": {
              "server_ready_s": 0.075,
              "first_request_wall_s": 6.513
            }
          },
          {
            "mode": "warmcache",
            "i": 3,
            "start_to_first_answer_s": 8.890463152,
            "r": {
              "server_ready_s": 0.027,
              "first_request_wall_s": 6.539
            }
          }
        ],
        "cold_median_s": 8.71,
        "warmcache_median_s": 8.8,
        "note": "container start -> server ready -> first 1-token answer; cold = GGUF evicted from page cache"
      },
      "model": {
        "gguf": "same file as Ollama qwen2.5:3b",
        "gguf_blob": "sha256:5ee4f07cdb9beadbbb293e85803c569b01bd37ed059d2715faa7bb405f31caa6"
      }
    },
    {
      "source_file": "results/mtplx_20261015T082058Z.json",
      "original_result_sha256": "bd9881d632c5a91173452449788705a6db5e091c38f72a3051692c080c588e1c",
      "version": "2.12.2",
      "started_utc": "2026-10-15T08:20:58Z",
      "finished_utc": "2026-10-15T08:21:19Z",
      "tool": "mtplx",
      "first_request": {},
      "model": {}
    },
    {
      "source_file": "results/ollama_20261015T081212Z.json",
      "original_result_sha256": "0b01434879b111546e179dede3a2ec0434816465fc3679ddda9a20e29a10e40c",
      "version": "0.40.4",
      "started_utc": "2026-10-15T08:12:12Z",
      "finished_utc": "2026-10-15T08:18:54Z",
      "tool": "ollama",
      "first_request": {
        "runs": [
          {
            "mode": "cold",
            "i": 1,
            "r": {
              "server_ready_s": 0.026,
              "first_request_wall_s": 7.891,
              "server_load_s": 7.191
            }
          },
          {
            "mode": "cold",
            "i": 2,
            "r": {
              "server_ready_s": 0.025,
              "first_request_wall_s": 6.269,
              "server_load_s": 5.616
            }
          },
          {
            "mode": "cold",
            "i": 3,
            "r": {
              "server_ready_s": 0.078,
              "first_request_wall_s": 6.652,
              "server_load_s": 5.876
            }
          },
          {
            "mode": "warmcache",
            "i": 1,
            "r": {
              "server_ready_s": 0.028,
              "first_request_wall_s": 5.346,
              "server_load_s": 4.608
            }
          },
          {
            "mode": "warmcache",
            "i": 2,
            "r": {
              "server_ready_s": 0.074,
              "first_request_wall_s": 7.986,
              "server_load_s": 6.676
            }
          },
          {
            "mode": "warmcache",
            "i": 3,
            "r": {
              "server_ready_s": 0.081,
              "first_request_wall_s": 5.289,
              "server_load_s": 4.603
            }
          }
        ],
        "cold_median_s": 6.652,
        "cold_server_load_median_s": 5.876,
        "warmcache_median_s": 5.346,
        "warmcache_server_load_median_s": 4.608,
        "note": "cold = model file evicted from page cache + fresh container; warmcache = fresh container, file still cached"
      },
      "model": {
        "tag": "qwen2.5:3b",
        "manifest_sha256": "cd232a7a8d590449a99eb8fe29b614ef9bf3624d2d17faa496e1135ba4188be0",
        "gguf_blob": "sha256:5ee4f07cdb9beadbbb293e85803c569b01bd37ed059d2715faa7bb405f31caa6",
        "gguf_size_mb": 1930,
        "pull_s": 0.4
      }
    },
    {
      "source_file": "results/ollama_20261016T084309Z.json",
      "original_result_sha256": "3d819fbb377328a9c4064f603579f860488578f158caa44ea86d5535c20705b8",
      "version": "0.40.5",
      "started_utc": "2026-10-16T08:43:09Z",
      "finished_utc": "2026-10-16T08:51:02Z",
      "tool": "ollama",
      "first_request": {
        "runs": [
          {
            "mode": "cold",
            "i": 1,
            "r": {
              "server_ready_s": 0.1,
              "first_request_wall_s": 13.77,
              "server_load_s": 13.031
            }
          },
          {
            "mode": "cold",
            "i": 2,
            "r": {
              "server_ready_s": 0.087,
              "first_request_wall_s": 7.343,
              "server_load_s": 6.632
            }
          },
          {
            "mode": "cold",
            "i": 3,
            "r": {
              "server_ready_s": 0.09,
              "first_request_wall_s": 7.8,
              "server_load_s": 7.141
            }
          },
          {
            "mode": "warmcache",
            "i": 1,
            "r": {
              "server_ready_s": 0.083,
              "first_request_wall_s": 5.035,
              "server_load_s": 4.337
            }
          },
          {
            "mode": "warmcache",
            "i": 2,
            "r": {
              "server_ready_s": 0.082,
              "first_request_wall_s": 5.014,
              "server_load_s": 4.334
            }
          },
          {
            "mode": "warmcache",
            "i": 3,
            "r": {
              "server_ready_s": 0.029,
              "first_request_wall_s": 4.974,
              "server_load_s": 4.332
            }
          }
        ],
        "cold_median_s": 7.8,
        "cold_server_load_median_s": 7.141,
        "warmcache_median_s": 5.014,
        "warmcache_server_load_median_s": 4.334,
        "note": "cold = model file evicted from page cache + fresh container; warmcache = fresh container, file still cached"
      },
      "model": {
        "tag": "qwen2.5:3b",
        "manifest_sha256": "cd232a7a8d590449a99eb8fe29b614ef9bf3624d2d17faa496e1135ba4188be0",
        "gguf_blob": "sha256:5ee4f07cdb9beadbbb293e85803c569b01bd37ed059d2715faa7bb405f31caa6",
        "gguf_size_mb": 1930,
        "pull_s": 0.4
      }
    }
  ]
}
