{
  "createdAt": "2026-10-19T07:19:40.280Z",
  "model": "gemini-3.8-flash",
  "promptSha256": "671686ccfd0eb25aa78227ff93279d7b706ee49816f0f5d573b7164a9c880eef",
  "policySha256": "d91f80874258df8fb7fd244c61a49925baca2240a18a63592dde3e9a27527ece",
  "testIds": [
    "cfpb-24292124",
    "cfpb-24488707",
    "cfpb-23915559",
    "cfpb-23973496",
    "cfpb-24128138",
    "cfpb-24025154",
    "cfpb-24161731",
    "cfpb-24237516",
    "cfpb-23805366",
    "cfpb-23788358",
    "cfpb-23990561",
    "cfpb-24190135",
    "cfpb-23758088",
    "cfpb-24679379",
    "cfpb-23964261",
    "cfpb-24699849",
    "cfpb-23915416",
    "cfpb-24526421",
    "cfpb-24014452",
    "cfpb-24158404",
    "cfpb-23820945",
    "cfpb-24148130",
    "cfpb-23897657",
    "cfpb-23764501"
  ],
  "testDataSha256": "c361cd8122ceec96b54a20ff13f6c3531066b3efcf794d9f156458e953a0703f",
  "maxCalls": 36,
  "maxOutputTokensPerCall": 1800,
  "maxAttempts": 1,
  "completedAt": "2026-10-19T07:21:50.057Z",
  "label": "Agreement with CFPB product-derived demo queues; not validated routing accuracy",
  "referenceAgreement": {
    "correct": 18,
    "total": 24,
    "rate": 0.75,
    "interval95": {
      "low": 0.5510017468789357,
      "high": 0.8800079652065766
    }
  },
  "rulesBaseline": {
    "correct": 12,
    "total": 24,
    "rate": 0.5
  },
  "failedCalls": 0,
  "p50GenerationMs": 3116,
  "p90GenerationMs": 4573,
  "providerCostUsd": null,
  "measuredTokens": {
    "input": 12229,
    "output": 4200,
    "reasoning": null,
    "callsWithCompleteUsage": 0,
    "callsWithUnknownUsage": 36,
    "callsReportingReasoning": 0
  },
  "autoDeliveryEnabled": false,
  "perQueue": {
    "cards": {
      "total": 6,
      "agreed": 1
    },
    "banking": {
      "total": 6,
      "agreed": 5
    },
    "credit": {
      "total": 6,
      "agreed": 6
    },
    "specialist": {
      "total": 6,
      "agreed": 6
    }
  },
  "cases": [
    {
      "id": "cfpb-24292124",
      "sourceDate": "2026-07-17",
      "referenceQueue": "cards",
      "predictedQueue": "cards",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24488707",
      "sourceDate": "2026-07-22",
      "referenceQueue": "cards",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23915559",
      "sourceDate": "2026-07-06",
      "referenceQueue": "cards",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23973496",
      "sourceDate": "2026-07-08",
      "referenceQueue": "cards",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24128138",
      "sourceDate": "2026-07-13",
      "referenceQueue": "cards",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24025154",
      "sourceDate": "2026-07-09",
      "referenceQueue": "cards",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24161731",
      "sourceDate": "2026-07-14",
      "referenceQueue": "banking",
      "predictedQueue": "banking",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24237516",
      "sourceDate": "2026-07-15",
      "referenceQueue": "banking",
      "predictedQueue": "banking",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23805366",
      "sourceDate": "2026-07-02",
      "referenceQueue": "banking",
      "predictedQueue": "cards",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23788358",
      "sourceDate": "2026-07-02",
      "referenceQueue": "banking",
      "predictedQueue": "banking",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23990561",
      "sourceDate": "2026-07-08",
      "referenceQueue": "banking",
      "predictedQueue": "banking",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24190135",
      "sourceDate": "2026-07-14",
      "referenceQueue": "banking",
      "predictedQueue": "banking",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23758088",
      "sourceDate": "2026-07-01",
      "referenceQueue": "credit",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24679379",
      "sourceDate": "2026-07-28",
      "referenceQueue": "credit",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23964261",
      "sourceDate": "2026-07-08",
      "referenceQueue": "credit",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24699849",
      "sourceDate": "2026-07-28",
      "referenceQueue": "credit",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23915416",
      "sourceDate": "2026-07-06",
      "referenceQueue": "credit",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24526421",
      "sourceDate": "2026-07-23",
      "referenceQueue": "credit",
      "predictedQueue": "credit",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24014452",
      "sourceDate": "2026-07-09",
      "referenceQueue": "specialist",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24158404",
      "sourceDate": "2026-07-14",
      "referenceQueue": "specialist",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23820945",
      "sourceDate": "2026-07-03",
      "referenceQueue": "specialist",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-24148130",
      "sourceDate": "2026-07-13",
      "referenceQueue": "specialist",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23897657",
      "sourceDate": "2026-07-06",
      "referenceQueue": "specialist",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    },
    {
      "id": "cfpb-23764501",
      "sourceDate": "2026-07-01",
      "referenceQueue": "specialist",
      "predictedQueue": "specialist",
      "needsClarification": true,
      "validated": true
    }
  ],
  "limitations": [
    "Small, balanced short-text convenience sample; not population representative.",
    "Reference labels are submitted products mapped to demo queues, not independent operations-owner truth.",
    "Dates separate development (June) from test (July); test text was inspected for privacy, not independently blind-labeled.",
    "Failures count as non-agreement. Intervals are illustrative record-level Wilson intervals; unrecognized correlated templates can make them optimistic.",
    "Latency percentiles cover validated test responses only. Token totals include provider-reported usage even for invalid responses; absent token fields are excluded and unknown-usage calls counted separately.",
    "Provider fees were not measured; no cost or ROI figure is claimed.",
    "No test threshold grants automatic delivery. Every case needs human approval."
  ],
  "payloadSha256": "7754c94368914bd3d4d5e33459967215628b6cd4b000f0e0b64b706c59b948ba"
}