{
  "title": "Richpanel customer support benchmark",
  "date": "2026-10-02",
  "status": "Launch v1 · full bank",
  "bank": "Launch v1",
  "judgeVersion": "2026-10-02-evidence-v9",
  "judgeModel": "gpt-6.1-sol",
  "attempts": 288,
  "cases": 72,
  "models": [
    {
      "key": "sol",
      "name": "GPT-6.1 Sol",
      "lab": "OpenAI",
      "modelId": "gpt-6.1-sol",
      "cases": 72,
      "fatal": 11,
      "fatalRate": 15.277777777777779,
      "score": 7.983333333333333,
      "process": 3.138888888888889,
      "quality": 0.8833333333333333,
      "cost": 0.17855972222222222,
      "credits": 0.35711944444444443,
      "tokens": 186385.97222222222,
      "inputTokens": 184986.22222222222,
      "outputTokens": 1399.75,
      "cacheReadTokens": 169682.44444444444,
      "cacheWriteTokens": 0,
      "usageSource": "Per-call usage and recorded provider rates for all 72 attempts."
    },
    {
      "key": "sonnet",
      "name": "Claude Sonnet 5.5",
      "lab": "Anthropic",
      "modelId": "claude-sonnet-5-5",
      "cases": 72,
      "fatal": 16,
      "fatalRate": 22.22222222222222,
      "score": 7.055555555555555,
      "process": 2.8055555555555554,
      "quality": 0.8277777777777778,
      "cost": 0.49197777777777785,
      "credits": 0.9839555555555557,
      "tokens": 217113.69444444444,
      "inputTokens": 216049.20833333334,
      "outputTokens": 1064.486111111111,
      "cacheReadTokens": 175227.55555555556,
      "cacheWriteTokens": 21157.625,
      "usageSource": "Per-call usage and recorded provider rates for all 72 attempts."
    },
    {
      "key": "gemini",
      "name": "Gemini 3 Flash",
      "lab": "Google",
      "modelId": "gemini-3-flash-preview",
      "cases": 72,
      "fatal": 16,
      "fatalRate": 22.22222222222222,
      "score": 6.541666666666667,
      "process": 2.3333333333333335,
      "quality": 0.6972222222222222,
      "cost": 0.1919951388888889,
      "credits": 0.3839902777777778,
      "tokens": 178140.23611111112,
      "inputTokens": 177611.13888888888,
      "outputTokens": 529.0972222222222,
      "cacheReadTokens": 139923.125,
      "cacheWriteTokens": 0,
      "usageSource": "Per-call usage and recorded provider rates for all 72 attempts."
    },
    {
      "key": "luna",
      "name": "GPT-6 Luna",
      "lab": "OpenAI",
      "modelId": "gpt-6-luna",
      "cases": 72,
      "fatal": 9,
      "fatalRate": 12.5,
      "score": 7.4222222222222225,
      "process": 2.4027777777777777,
      "quality": 0.8166666666666667,
      "cost": 0.12556875,
      "credits": 0.2511375,
      "tokens": 267417.31944444444,
      "inputTokens": 267099.55555555556,
      "outputTokens": 317.7638888888889,
      "cacheReadTokens": 252036.15277777778,
      "cacheWriteTokens": 0,
      "usageSource": "Per-call usage and recorded provider rates for all 72 attempts."
    }
  ],
  "calibration": {
    "cases": 3,
    "judgmentsPerCase": 3,
    "unstableCases": 1
  },
  "measurementUnit": "One replay attempt from a captured moment, not the full conversation lifecycle.",
  "pricing": {
    "term": "annual",
    "commitment": "committed",
    "pricePerCredit": 0.5,
    "sourceDigest": "c20f26f9bbe2780d728f70ac1eb3072859b1cd9f3799532ff23db618386eec46"
  },
  "costBasis": "Recorded agent usage converted to credits using Richpanel billing rules, including model markup and per-call rounding, priced at the annual committed rate. These replays did not charge a customer.",
  "costExclusions": [
    "Judge calls",
    "Retrieval",
    "Tools",
    "Infrastructure",
    "Seats",
    "Minimum purchases and unused committed credits"
  ],
  "sourceDigest": "f5c6450b0f1735478d185a00450311c04a7f049fb3e4849404f1ada0e6c3030b",
  "examples": [
    {
      "id": "complete-product-answer",
      "label": "Product question",
      "title": "A correct answer still needs the required follow-through.",
      "headline": "A correct answer is only part of the job.",
      "task": "Answer a product question using the human agent’s supplied explanation.",
      "situation": "A customer asks for a product detail the brand does not publish. A human agent supplies the explanation and guarantee to include.",
      "evidence": "The human supplies the answer. The required exact product tag is absent from the tag search results.",
      "expected": "Use the supplied explanation, check the required tags, and record any missing tag.",
      "finding": "All four answered the question. Sol also completed the available tagging steps and documented the missing tag.",
      "outcomes": [
        {
          "model": "sol",
          "score": 10,
          "fatal": false,
          "description": "Answered correctly, applied the available inquiry tag, and documented the missing product tag."
        },
        {
          "model": "luna",
          "score": 9,
          "fatal": false,
          "description": "Answered correctly, but skipped the required tag lookup and tagging."
        },
        {
          "model": "sonnet",
          "score": 9,
          "fatal": false,
          "description": "Answered correctly, but omitted the required inquiry and product tags and their lookup."
        },
        {
          "model": "gemini",
          "score": 7.8,
          "fatal": false,
          "description": "Answered correctly, but missed the missing-tag note and described its sent reply as a draft."
        }
      ]
    },
    {
      "id": "urgent-cancellation",
      "label": "Urgent cancellation",
      "title": "An urgent cancellation needs a complete handoff.",
      "headline": "Stop the next shipment before it leaves.",
      "task": "Handle an urgent cancellation request for a paid, unfulfilled subscription order.",
      "situation": "A customer asks to cancel an upcoming order after earlier pause requests. The order is paid but has not shipped.",
      "evidence": "The order is unfulfilled. Cancellation requires human action; no completed cancellation or pause is confirmed.",
      "expected": "Mark the case urgent, leave an actionable handoff, and keep it open without promising cancellation.",
      "finding": "All four preserved human review. Sol followed the process most completely; the others lost points for timing, tagging, or handoff details.",
      "outcomes": [
        {
          "model": "sol",
          "score": 9.8,
          "fatal": false,
          "description": "Marked urgent first, applied the required tags, and left a clear handoff. The reply’s tone was overly administrative."
        },
        {
          "model": "luna",
          "score": 8.8,
          "fatal": false,
          "description": "Left a useful handoff, but marked priority after replying and searched for tags without applying them."
        },
        {
          "model": "sonnet",
          "score": 8.8,
          "fatal": false,
          "description": "Preserved urgent review, but missed required tags and stated that no pause had occurred without verification."
        },
        {
          "model": "gemini",
          "score": 8.8,
          "fatal": false,
          "description": "Preserved review, but marked priority late, missed a subscription tag, and blurred pause and cancellation instructions."
        }
      ]
    },
    {
      "id": "preserve-human-handoff",
      "label": "Human handoff",
      "title": "Return a human-owned follow-up to the agent.",
      "headline": "The customer has answered the human agent.",
      "task": "Handle account information supplied in response to a human agent’s request.",
      "situation": "A customer supplies details a human agent requested to investigate a refund and subscription issue. That agent still owns the conversation.",
      "evidence": "The latest human reply requests the information. The customer’s response requires an internal handoff, not a new AI-led exchange.",
      "expected": "Pass the details and outstanding questions back in an internal note. Keep the conversation open without sending a new AI reply.",
      "finding": "Sol, Luna, and Sonnet preserved the handoff. Gemini restarted verification and omitted the required internal note, triggering a fatal error.",
      "outcomes": [
        {
          "model": "sol",
          "score": 10,
          "fatal": false,
          "description": "Saved the supplied details and unresolved questions in a useful handoff note. Left the case open without replying."
        },
        {
          "model": "luna",
          "score": 10,
          "fatal": false,
          "description": "Saved a useful note requesting account, refund, and subscription review. Left the case open without replying."
        },
        {
          "model": "sonnet",
          "score": 10,
          "fatal": false,
          "description": "Named the owning agent and documented the next steps. Left the case open without replying."
        },
        {
          "model": "gemini",
          "score": 0,
          "fatal": true,
          "description": "Requested verification twice and replied to the customer, but never created the required handoff note."
        }
      ]
    }
  ]
}