{ "schema_version": 1, "measured_at": "2026-09-17T15:57:03.184Z", "mode": "live", "split": "holdout", "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "rubric_sha256": "d9550c3c29d1dfb783ad58ee7749c88ccf427d33ba755b59815a15a23e01f565", "corpus_sha256": "66919e117ca89fcdb2ace8ccda13641c12919625a5f16b25109d3ab2720d729a", "confusion_matrix": { "appears_supported": { "appears_supported": 10, "scope_gap": 0, "contradicted": 0, "insufficient_context": 0, "error": 0 }, "scope_gap": { "appears_supported": 0, "scope_gap": 10, "contradicted": 0, "insufficient_context": 0, "error": 0 }, "contradicted": { "appears_supported": 0, "scope_gap": 0, "contradicted": 10, "insufficient_context": 0, "error": 0 }, "insufficient_context": { "appears_supported": 0, "scope_gap": 0, "contradicted": 0, "insufficient_context": 10, "error": 0 } }, "false_support": 0, "inadequate_flagged": 20, "adequate_correct": 10, "insufficient_correct": 10, "errors": 0, "pilot_gate": "PASS", "latency_ms": { "p50": 178, "p95": 347 }, "usage": { "input_tokens": 28656, "output_tokens": 2300 }, "low_confidence_results": 5, "failures": [], "rows": [ { "id": "holdout-01", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 1, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 710, "output_tokens": 56 }, "latency_ms": 396 } }, { "id": "holdout-02", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.97, "probabilities": { "appears_supported": 0, "scope_gap": 0.99, "contradicted": 0, "insufficient_context": 0.01 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 710, "output_tokens": 56 }, "latency_ms": 317 } }, { "id": "holdout-03", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.9, "probabilities": { "appears_supported": 0, "scope_gap": 0.92, "contradicted": 0.07, "insufficient_context": 0.01 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 704, "output_tokens": 56 }, "latency_ms": 171 } }, { "id": "holdout-04", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.99, "probabilities": { "appears_supported": 0, "scope_gap": 1, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 707, "output_tokens": 56 }, "latency_ms": 175 } }, { "id": "holdout-05", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.99, "probabilities": { "appears_supported": 0, "scope_gap": 0.99, "contradicted": 0.01, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 738, "output_tokens": 56 }, "latency_ms": 155 } }, { "id": "holdout-06", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 1, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 718, "output_tokens": 56 }, "latency_ms": 138 } }, { "id": "holdout-07", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.94, "probabilities": { "appears_supported": 0.01, "scope_gap": 0.96, "contradicted": 0, "insufficient_context": 0.03 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 716, "output_tokens": 56 }, "latency_ms": 149 } }, { "id": "holdout-08", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.95, "probabilities": { "appears_supported": 0, "scope_gap": 0.96, "contradicted": 0.04, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 717, "output_tokens": 56 }, "latency_ms": 144 } }, { "id": "holdout-09", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.99, "probabilities": { "appears_supported": 0, "scope_gap": 1, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 713, "output_tokens": 56 }, "latency_ms": 159 } }, { "id": "holdout-10", "expected": "scope_gap", "verdict": "scope_gap", "result": { "advisory": true, "verdict": "scope_gap", "confidence": 0.95, "probabilities": { "appears_supported": 0, "scope_gap": 0.96, "contradicted": 0.04, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 708, "output_tokens": 56 }, "latency_ms": 152 } }, { "id": "holdout-11", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 712, "output_tokens": 58 }, "latency_ms": 155 } }, { "id": "holdout-12", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 718, "output_tokens": 58 }, "latency_ms": 151 } }, { "id": "holdout-13", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 716, "output_tokens": 58 }, "latency_ms": 148 } }, { "id": "holdout-14", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 715, "output_tokens": 58 }, "latency_ms": 144 } }, { "id": "holdout-15", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 719, "output_tokens": 58 }, "latency_ms": 236 } }, { "id": "holdout-16", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 717, "output_tokens": 58 }, "latency_ms": 145 } }, { "id": "holdout-17", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 716, "output_tokens": 58 }, "latency_ms": 310 } }, { "id": "holdout-18", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 720, "output_tokens": 58 }, "latency_ms": 347 } }, { "id": "holdout-19", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 707, "output_tokens": 58 }, "latency_ms": 155 } }, { "id": "holdout-20", "expected": "contradicted", "verdict": "contradicted", "result": { "advisory": true, "verdict": "contradicted", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 1, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 721, "output_tokens": 58 }, "latency_ms": 143 } }, { "id": "holdout-21", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.84, "probabilities": { "appears_supported": 0.88, "scope_gap": 0.06, "contradicted": 0, "insufficient_context": 0.06 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 734, "output_tokens": 57 }, "latency_ms": 178 } }, { "id": "holdout-22", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.97, "probabilities": { "appears_supported": 0.98, "scope_gap": 0.02, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 748, "output_tokens": 57 }, "latency_ms": 180 } }, { "id": "holdout-23", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.99, "probabilities": { "appears_supported": 1, "scope_gap": 0, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 729, "output_tokens": 57 }, "latency_ms": 170 } }, { "id": "holdout-24", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.94, "probabilities": { "appears_supported": 0.95, "scope_gap": 0.04, "contradicted": 0, "insufficient_context": 0.01 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 740, "output_tokens": 57 }, "latency_ms": 158 } }, { "id": "holdout-25", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.96, "probabilities": { "appears_supported": 0.98, "scope_gap": 0.01, "contradicted": 0, "insufficient_context": 0.01 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 764, "output_tokens": 57 }, "latency_ms": 189 } }, { "id": "holdout-26", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.94, "probabilities": { "appears_supported": 0.96, "scope_gap": 0.04, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 735, "output_tokens": 57 }, "latency_ms": 502 } }, { "id": "holdout-27", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.98, "probabilities": { "appears_supported": 0.99, "scope_gap": 0.01, "contradicted": 0, "insufficient_context": 0 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 736, "output_tokens": 57 }, "latency_ms": 188 } }, { "id": "holdout-28", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.62, "probabilities": { "appears_supported": 0.72, "scope_gap": 0.08, "contradicted": 0.15, "insufficient_context": 0.05 }, "requires_review": true, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 732, "output_tokens": 57 }, "latency_ms": 219 } }, { "id": "holdout-29", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.94, "probabilities": { "appears_supported": 0.95, "scope_gap": 0.04, "contradicted": 0, "insufficient_context": 0.01 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 733, "output_tokens": 57 }, "latency_ms": 145 } }, { "id": "holdout-30", "expected": "appears_supported", "verdict": "appears_supported", "result": { "advisory": true, "verdict": "appears_supported", "confidence": 0.98, "probabilities": { "appears_supported": 0.98, "scope_gap": 0.01, "contradicted": 0, "insufficient_context": 0.01 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 724, "output_tokens": 57 }, "latency_ms": 260 } }, { "id": "holdout-31", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.98, "probabilities": { "appears_supported": 0, "scope_gap": 0.01, "contradicted": 0, "insufficient_context": 0.99 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 689, "output_tokens": 59 }, "latency_ms": 198 } }, { "id": "holdout-32", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 0, "insufficient_context": 1 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 681, "output_tokens": 59 }, "latency_ms": 256 } }, { "id": "holdout-33", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 0, "insufficient_context": 1 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 708, "output_tokens": 59 }, "latency_ms": 178 } }, { "id": "holdout-34", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.7, "probabilities": { "appears_supported": 0.01, "scope_gap": 0.22, "contradicted": 0, "insufficient_context": 0.77 }, "requires_review": true, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 695, "output_tokens": 59 }, "latency_ms": 220 } }, { "id": "holdout-35", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.38, "probabilities": { "appears_supported": 0, "scope_gap": 0.46, "contradicted": 0.01, "insufficient_context": 0.53 }, "requires_review": true, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 695, "output_tokens": 59 }, "latency_ms": 232 } }, { "id": "holdout-36", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.68, "probabilities": { "appears_supported": 0.12, "scope_gap": 0.12, "contradicted": 0, "insufficient_context": 0.76 }, "requires_review": true, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 697, "output_tokens": 59 }, "latency_ms": 201 } }, { "id": "holdout-37", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.73, "probabilities": { "appears_supported": 0, "scope_gap": 0.12, "contradicted": 0.07, "insufficient_context": 0.81 }, "requires_review": true, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 697, "output_tokens": 59 }, "latency_ms": 129 } }, { "id": "holdout-38", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.85, "probabilities": { "appears_supported": 0, "scope_gap": 0.11, "contradicted": 0, "insufficient_context": 0.89 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 706, "output_tokens": 59 }, "latency_ms": 179 } }, { "id": "holdout-39", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 0.92, "probabilities": { "appears_supported": 0.01, "scope_gap": 0.06, "contradicted": 0, "insufficient_context": 0.9299999999999999 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 705, "output_tokens": 59 }, "latency_ms": 194 } }, { "id": "holdout-40", "expected": "insufficient_context", "verdict": "insufficient_context", "result": { "advisory": true, "verdict": "insufficient_context", "confidence": 1, "probabilities": { "appears_supported": 0, "scope_gap": 0, "contradicted": 0, "insufficient_context": 1 }, "requires_review": false, "model": "jev-1.13.0", "rubric_version": "check-proof-v1", "usage": { "input_tokens": 706, "output_tokens": 59 }, "latency_ms": 195 } } ] }