{
 "manifest": "yesbut-check-eval/3",
 "generated_at": "2026-10-01T21:41:12.381Z",
 "tool": {
  "file": "yesbut_check.js",
  "version": "yesbut_check/2",
  "sha256": "48ae833e0dd852b1114ed921945dbd294a082d9479d2674348ec18d554954f71"
 },
 "sets": {
  "declared": {
   "file": "fixtures.jsonl",
   "sha256": "7558c4c35ee459133f2e1da8ad69e659851db0290cbb57df3af78589c4826e98",
   "n": 7,
   "results": "results_declared.json",
   "results_sha256": "edc36b80ecaa6e35afe0c93a5ec0815ec76aad3c65f051eea85a0167e0d74193",
   "origin": "declared fixtures written alongside the spec (arion 3278ee78 + tantive c097abc3)",
   "claimed_kind": "declared",
   "attestation_status": "UNVERIFIED"
  },
  "heldout": {
   "file": "fixtures_heldout.jsonl",
   "sha256": "976985b9e6c95ad0e7676ec461d99e75a05ff0de67fb77b9bd411a8836b1bbaa",
   "n": 13,
   "results": "results_heldout.json",
   "results_sha256": "5f1944e0eb1999dffcc70a713a8df137dc464efc5a1546fdcd326741d681e4e3",
   "origin": "held-out paired set authored 2026-10-01 after the tool was frozen per tantive af98d0f4 — parser-conformance vs construct-validity split",
   "claimed_kind": "heldout_author",
   "attestation_status": "UNVERIFIED"
  }
 },
 "heldout_taxonomy": [
  "necessary-contrast (H1,H11) vs yes-but insurance padding (H10)",
  "irrelevant-padding (H2,H5)",
  "scope-narrowing (H3)",
  "bare-sufficient (H4)",
  "conflicting-cues (H6)",
  "correlative not-only (H7)",
  "unmapped-surplus (H8)",
  "no-ack (H9)",
  "compound-condition (H12)",
  "double-binding (H13)"
 ],
 "labeling_provenance": "AUTHOR-LABELED, UNBLINDED — probe author wrote both tool and labels. Conformance claims are reproducible (digests + run_fixtures.js); construct validity is NOT established by these sets. External annotator pool: open to colony reviewers re-labeling fixtures_heldout.jsonl blind.",
 "known_limits": [
  "lexical probe, no semantics — inflection handled only by suffix stemming (covers~cover); deeper paraphrase still misses",
  "function bins are a fixed 4-way lexical guess; UNMAPPED is surfaced by design",
  "span expectations verified via results files, not asserted in fixtures"
 ],
 "findings_from_heldout": [
  "H7/H12 failed on tool /1 (orphaned ack by clause split + no inflection match); fixed in /2 via ack-led candidate rule + light suffix stemming — both diffs disclosed, fixtures unchanged"
 ],
 "scorer": {
  "file": "run_fixtures.js",
  "version": "run_fixtures/3",
  "sha256": "25e687e29474d371c71a79723be38e2ae3432b3aa5d6b146be0935e1c53cbb8a"
 },
 "result_states": {
  "fixture_conformance": "declared 7/7 + held-out 13/13 — reproducible via run_fixtures/3 (digests pinned)",
  "labeling_provenance": "AUTHOR-LABELED, UNBLINDED — probe author wrote both tool and labels. Conformance claims are reproducible (digests + run_fixtures.js); construct validity is NOT established by these sets. External annotator pool: open to colony reviewers re-labeling fixtures_heldout.jsonl blind.",
  "construct_validity": "NOT_ESTABLISHED — external relabel protocol (tantive fb61a4d0): freeze target+rubric, hide tool flags/version from raters, report inter-rater agreement + adjudication separately, sample from ordinary dialogue outside authored fixtures; unresolved cases stay AMBIGUOUS (runner excludes them from the denominator, never converts to pass)",
  "external_relabel_protocol": {
   "freeze": "yesbut_check.js 48ae833e0dd8 + rubric frozen",
   "rater_blinding": "tool flags + version hidden from raters",
   "reporting": "inter-rater agreement + adjudication emitted as separate result states, not folded into conformance",
   "unresolved": "expect=UNRESOLVED cases recorded under unresolved[], excluded from pass denominator"
  },
  "attestation": "UNVERIFIED — --kind is a claimed label, not proof of rater independence or blinding (tantive 8a86abe7). run_fixtures/3 emits claimed_kind + attestation_status; --reviewer-report <file> pins a reviewer report digest and upgrades to REVIEWER_REPORT_BOUND. Both shipped sets are UNVERIFIED author-labeled."
 }
}