{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://medgauge.org/schemas/0.1.0/result.schema.json",
  "title": "result",
  "description": "A site-reviewed result record under methodology 0.1.0: benchmark version, model snapshot, harness version, run conditions, the three runs of every task with error attribution, scores by subset and dimension with their intervals, the exam outcome and its lower bound, vetoes by category, amounts with the price table version, and the hash of the run records.",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "id": {
      "type": "string",
      "pattern": "^res-\\d{4}-\\d{2}-\\d{2}-[a-z0-9.-]+$"
    },
    "methodology_version": {
      "type": "string",
      "pattern": "^\\d+\\.\\d+\\.\\d+$",
      "description": "Methodology version in force when the record was made"
    },
    "benchmark": {
      "type": "string",
      "pattern": "^bm-[a-z0-9-]+$"
    },
    "benchmark_version": {
      "type": "string",
      "pattern": "^\\d+\\.\\d+\\.\\d+$"
    },
    "model": {
      "type": "string",
      "minLength": 1
    },
    "model_snapshot": {
      "type": "string",
      "minLength": 1
    },
    "harness": {
      "type": "string",
      "pattern": "^hx-[a-z0-9-]+$"
    },
    "harness_version": {
      "type": "string",
      "pattern": "^\\d+\\.\\d+\\.\\d+$",
      "description": "registry/harnesses/<harness>/<harness_version>.yaml must exist"
    },
    "run_conditions": {
      "type": "object",
      "properties": {
        "reasoning_effort": {
          "type": "string"
        },
        "max_steps": {
          "type": "integer",
          "minimum": 1
        },
        "budget_tokens": {
          "type": "integer",
          "minimum": 1
        },
        "container_image_digest": {
          "type": "string",
          "pattern": "^sha256:[0-9a-f]{64}$"
        },
        "tool_permissions": {
          "type": "string",
          "enum": [
            "readonly",
            "workspace_write",
            "full"
          ]
        },
        "allow_internet": {
          "type": "boolean",
          "const": false
        },
        "service_fingerprint": {
          "type": "string",
          "minLength": 1
        },
        "max_output_tokens": {
          "type": "integer",
          "minimum": 1
        },
        "temperature": {
          "type": "number",
          "minimum": 0,
          "maximum": 2
        },
        "command_timeout_sec": {
          "type": "integer",
          "minimum": 1
        },
        "wall_clock_sec": {
          "type": "integer",
          "minimum": 1
        },
        "tools": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        },
        "target_runs_per_task": {
          "type": "integer",
          "const": 3,
          "description": "k = 3 runs per task; a run error is a failed run, never re-run (G1, G7)"
        },
        "max_steps_note": {
          "type": "string"
        }
      },
      "required": [
        "service_fingerprint",
        "tool_permissions",
        "target_runs_per_task",
        "allow_internet"
      ],
      "additionalProperties": false
    },
    "run_outcomes": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/task_outcome"
      },
      "minItems": 1,
      "description": "Every run of every task, keyed by opaque task key, so that every published figure can be recomputed from the record (G6)."
    },
    "scores": {
      "type": "object",
      "properties": {
        "n_tasks_total": {
          "type": "integer",
          "minimum": 1
        },
        "veto_hits": {
          "type": "object",
          "additionalProperties": {
            "type": "integer",
            "minimum": 0
          },
          "description": "Veto hits keyed by the X-domain constraint involved"
        },
        "by_subset": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "calib": {
              "$ref": "#/$defs/subset_scores"
            },
            "dev": {
              "$ref": "#/$defs/subset_scores"
            },
            "holdout": {
              "$ref": "#/$defs/subset_scores"
            }
          },
          "required": [],
          "description": "Subsets are reported separately and never pooled; leaderboard figures come from holdout",
          "minProperties": 1
        },
        "assertion_veto_hits": {
          "type": "object",
          "description": "Grading-reference veto hits, keyed by the X-domain constraint each vetoing reference maps to",
          "additionalProperties": {
            "type": "integer",
            "minimum": 0
          }
        },
        "dimensions": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "m3_efficiency": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "action_definition": {
                  "type": "string",
                  "minLength": 1,
                  "description": "The harness version's own definition of one action (G8); action figures compare only within this harness version"
                },
                "action_ratio": {
                  "$ref": "#/$defs/dispersion"
                },
                "amount_ratio": {
                  "$ref": "#/$defs/dispersion"
                },
                "cost_per_success": {
                  "oneOf": [
                    {
                      "type": "number",
                      "minimum": 0
                    },
                    {
                      "const": "not_available"
                    }
                  ],
                  "description": "Total amount over all runs / successful runs, averaged over tasks; tasks with no success are not_available and stay out of the mean"
                },
                "median_tool_calls": {
                  "type": "number",
                  "minimum": 0
                },
                "median_tokens": {
                  "type": "number",
                  "minimum": 0
                },
                "median_wall_clock_sec": {
                  "type": "number",
                  "minimum": 0
                }
              },
              "required": [
                "action_definition"
              ],
              "description": "M3, on passing runs only except cost_per_success"
            },
            "m5_optimality": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "redundancy_rate": {
                  "type": "number",
                  "description": "(actions − |R|) / |R|, task median then family mean; may be negative when the harness's action is coarser than the declared granularity; compared only within one harness version"
                },
                "order_violations": {
                  "type": "number",
                  "minimum": 0
                },
                "tool_deviations": {
                  "type": "number",
                  "minimum": 0
                },
                "n_tasks": {
                  "type": "integer",
                  "minimum": 0
                }
              },
              "description": "M5, on passing runs of tasks whose metadata carries a required action set"
            },
            "m7_quality_control": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "defect_recall": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "defect_precision": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "defect_f1": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1,
                  "description": "Pooled over seeded defects within the family"
                },
                "defect_f1_task_equal": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1,
                  "description": "Task-weighted version, for comparison"
                },
                "self_check_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1,
                  "description": "Runs with a checking action before submission / runs"
                },
                "traceability_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "evidence_error_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1,
                  "description": "Cited but not supporting"
                },
                "by_defect_class_recall": {
                  "type": "object",
                  "additionalProperties": {
                    "type": "number",
                    "minimum": 0,
                    "maximum": 1
                  }
                }
              },
              "description": "M7"
            },
            "m8_exploration": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "appropriate_abstention_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "over_abstention_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "overreach_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1,
                  "description": "Counted under the unsafe_advice veto category of M6"
                },
                "brier": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1,
                  "description": "Only where artifacts carry their own confidence field; no confidence is required of a subject"
                },
                "n_undecidable_points": {
                  "type": "integer",
                  "minimum": 0
                },
                "n_decidable_points": {
                  "type": "integer",
                  "minimum": 0
                }
              },
              "description": "M8"
            },
            "m9_auditability": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "regrade_agreement": {
                  "const": 1,
                  "description": "Re-grading every stored artifact reproduces every checkpoint result; a record that does not is not published"
                },
                "trajectory_completeness": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "env_delta": {
                  "$ref": "#/$defs/signed_and_absolute"
                }
              },
              "required": [
                "regrade_agreement"
              ],
              "description": "M9"
            },
            "m10_robustness": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "robustness_gap": {
                  "$ref": "#/$defs/signed_and_absolute"
                },
                "flip_rate": {
                  "type": "number",
                  "minimum": 0,
                  "maximum": 1
                },
                "mcnemar": {
                  "type": "object",
                  "additionalProperties": false,
                  "properties": {
                    "b": {
                      "type": "integer",
                      "minimum": 0
                    },
                    "c": {
                      "type": "integer",
                      "minimum": 0
                    },
                    "p_value": {
                      "type": "number",
                      "minimum": 0,
                      "maximum": 1
                    }
                  },
                  "required": [
                    "b",
                    "c",
                    "p_value"
                  ],
                  "description": "Exact McNemar test on tasks folded to 0/1 by pass^3 (original against variant); b and c are the discordant counts"
                },
                "paired_bootstrap": {
                  "$ref": "#/$defs/bca_bootstrap"
                },
                "n_pairs": {
                  "type": "integer",
                  "minimum": 0
                }
              },
              "description": "M10, on tasks with perturbation variants"
            }
          },
          "description": "Dimensions M3 and M5–M10 (methodology section 6), each reported on its own; never combined into a total. A dimension a family's metadata does not support is absent, never 0."
        }
      },
      "required": [
        "n_tasks_total",
        "veto_hits",
        "by_subset"
      ],
      "additionalProperties": false
    },
    "board": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "letters": {
          "type": "array",
          "items": {
            "type": "string",
            "pattern": "^[a-z]$"
          },
          "minItems": 1,
          "uniqueItems": true,
          "description": "Compact letter display: every group letter the row belongs to. Rows sharing a letter are not shown to differ; groups may overlap"
        },
        "comparison": {
          "const": "paired_holm",
          "description": "Pairwise paired comparisons on the same tasks, Holm-adjusted"
        },
        "key": {
          "const": "pass_pow_3"
        },
        "secondary_key": {
          "const": "cost_per_success",
          "description": "Order within a letter; rows without an amount take no part in it"
        },
        "as_of": {
          "type": "string",
          "format": "date"
        },
        "n_rows": {
          "type": "integer",
          "minimum": 1
        }
      },
      "required": [
        "letters",
        "comparison",
        "key",
        "as_of",
        "n_rows"
      ],
      "description": "Position on the holdout board of the benchmark version (section 8.2). Recomputed when rows are added."
    },
    "cost": {
      "type": "object",
      "properties": {
        "tokens_in": {
          "type": "integer",
          "minimum": 0
        },
        "tokens_out": {
          "type": "integer",
          "minimum": 0
        },
        "price_table_version": {
          "type": "string",
          "description": "Version of the price table used; the table is published with the result"
        },
        "wall_time_sec": {
          "type": "number",
          "minimum": 0
        },
        "median_steps": {
          "type": "number",
          "minimum": 0,
          "description": "Median action count per run under the harness's own action definition"
        },
        "median_tool_calls": {
          "type": "number",
          "minimum": 0
        },
        "median_tokens": {
          "type": "number",
          "minimum": 0
        },
        "amount": {
          "type": "number",
          "minimum": 0,
          "description": "Total over all runs, in the price table's currency"
        },
        "currency": {
          "type": "string",
          "pattern": "^[A-Z]{3}$"
        },
        "amount_applicable": {
          "type": "boolean",
          "description": "false for self-hosted and open-weight subjects: no price table, no amount; the page shows 'amount not applicable'"
        }
      },
      "required": [
        "amount_applicable"
      ],
      "additionalProperties": false,
      "description": "M3 amounts and usage. Amounts are converted with a versioned price table published with the result; self-hosted and open-weight subjects report no amount.",
      "allOf": [
        {
          "if": {
            "properties": {
              "amount_applicable": {
                "const": true
              }
            }
          },
          "then": {
            "required": [
              "amount",
              "currency",
              "price_table_version"
            ]
          },
          "else": {
            "not": {
              "anyOf": [
                {
                  "required": [
                    "amount"
                  ]
                },
                {
                  "required": [
                    "price_table_version"
                  ]
                }
              ]
            }
          }
        }
      ]
    },
    "voided_tasks": {
      "type": "array",
      "description": "Task versions voided for every subject under the void-and-retest rule, with the attribution of the error that voided them",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "task_key": {
            "type": "string",
            "pattern": "^tk-[0-9a-f]{12,64}$"
          },
          "task_version": {
            "type": "string",
            "pattern": "^\\d+\\.\\d+\\.\\d+$"
          },
          "attribution": {
            "type": "string",
            "enum": [
              "task",
              "environment"
            ]
          }
        },
        "required": [
          "task_key",
          "task_version",
          "attribution"
        ]
      }
    },
    "failure_attribution": {
      "type": "object",
      "properties": {
        "missed": {
          "type": "integer",
          "minimum": 0
        },
        "overreported": {
          "type": "integer",
          "minimum": 0
        },
        "other": {
          "type": "integer",
          "minimum": 0
        },
        "step_limit": {
          "type": "integer",
          "minimum": 0
        },
        "fabricated_evidence": {
          "type": "integer",
          "minimum": 0
        },
        "format": {
          "type": "integer",
          "minimum": 0
        },
        "input_modified": {
          "type": "integer",
          "minimum": 0
        },
        "corrected_value": {
          "type": "integer",
          "minimum": 0
        },
        "followed_injection": {
          "type": "integer",
          "minimum": 0
        }
      },
      "additionalProperties": false,
      "description": "Failed runs by the reason the verifier recorded"
    },
    "run_records_hash": {
      "type": "string",
      "pattern": "^sha256:[0-9a-f]{64}$"
    },
    "trajectories_archived": {
      "type": "boolean"
    },
    "closed_checkpoint_detail": {
      "type": "object",
      "additionalProperties": false,
      "description": "Pointer to the per-checkpoint detail held on the closed side. The public record carries only counts and a hash; the detail (checkpoint id, pass/fail, linked grading reference) stays in the private grading repository.",
      "properties": {
        "stored_in": {
          "const": "closed"
        },
        "sha256": {
          "type": "string",
          "pattern": "^sha256:[0-9a-f]{64}$"
        },
        "n_checkpoints": {
          "type": "integer",
          "minimum": 0
        },
        "reference_set_version": {
          "type": "string",
          "description": "Version of the grading reference set the checkpoints were built against"
        },
        "n_reference_linked": {
          "type": "integer",
          "minimum": 0
        }
      },
      "required": [
        "stored_in",
        "sha256",
        "n_checkpoints"
      ]
    },
    "submitted_by": {
      "type": "object",
      "properties": {
        "org": {
          "type": "string",
          "minLength": 1
        },
        "contact": {
          "type": "string"
        },
        "verified": {
          "type": "boolean"
        }
      },
      "required": [
        "org",
        "verified"
      ],
      "additionalProperties": false
    },
    "verified_by_site": {
      "type": "boolean"
    },
    "verification_state": {
      "type": "string",
      "enum": [
        "site_verified",
        "community_unverified"
      ],
      "description": "site_verified = run or re-graded by the site and fully reviewed; community_unverified = submitted by a third party and not reviewed. Only site_verified records appear on the leaderboard."
    },
    "verification_note": {
      "type": "string"
    },
    "review_record": {
      "type": "string"
    },
    "created_at": {
      "type": "string",
      "format": "date-time"
    },
    "run_window": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "from": {
          "type": "string",
          "format": "date"
        },
        "to": {
          "type": "string",
          "format": "date"
        }
      },
      "required": [
        "from",
        "to"
      ]
    },
    "run_mode": {
      "type": "string",
      "enum": [
        "site_run",
        "submitted_run_regrading"
      ],
      "description": "site_run = run by the initiative on its own infrastructure; submitted_run_regrading = run by an enrolled party on a watermarked task package, the returned artifacts and run records re-graded by the initiative"
    }
  },
  "required": [
    "id",
    "run_outcomes",
    "methodology_version",
    "benchmark",
    "benchmark_version",
    "model",
    "model_snapshot",
    "harness",
    "harness_version",
    "run_conditions",
    "scores",
    "cost",
    "run_records_hash",
    "submitted_by",
    "verified_by_site",
    "created_at",
    "run_mode"
  ],
  "allOf": [
    {
      "if": {
        "properties": {
          "verified_by_site": {
            "const": true
          }
        }
      },
      "then": {
        "required": [
          "review_record",
          "verification_note"
        ]
      }
    }
  ],
  "$defs": {
    "rate": {
      "type": "number",
      "minimum": 0,
      "maximum": 1
    },
    "veto_category": {
      "type": "string",
      "enum": [
        "fabricated_evidence",
        "altered_input",
        "corrected_value",
        "executed_injection",
        "unsafe_advice"
      ],
      "description": "Veto categories of dimension M6 (methodology section 6): fabricated evidence, altered input, a corrected value substituted, a planted instruction executed, unsafe advice (including a definite conclusion at an undecidable point, M8)."
    },
    "error_source": {
      "type": "string",
      "enum": [
        "model",
        "agent",
        "harness",
        "application",
        "provider",
        "task",
        "environment"
      ],
      "description": "Where a run error is attributed (general rule G7). model, agent, harness, application and provider are on the subject's side: the run is recorded as failed and counted in the published error rate. task and environment are on the initiative's side: the task version is voided for every subject under the void-and-retest rule (section 8.4)."
    },
    "clopper_pearson": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "method": {
          "const": "clopper_pearson"
        },
        "level": {
          "const": 0.95
        },
        "lower": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "Two-sided 95% lower limit"
        },
        "upper": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "Two-sided 95% upper limit"
        },
        "one_sided_lower": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "One-sided 95% lower bound; used for exam claims and levels. With no failure among n tasks it equals 0.05^(1/n)."
        }
      },
      "required": [
        "method",
        "level",
        "lower",
        "upper",
        "one_sided_lower"
      ],
      "description": "Clopper–Pearson exact interval for a 0/1 task quantity (task passed, pass@3, pass^3), over tasks (section 8.1)."
    },
    "bca_bootstrap": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "method": {
          "const": "bca_task_bootstrap"
        },
        "level": {
          "const": 0.95
        },
        "resamples": {
          "const": 10000
        },
        "seed": {
          "type": "integer",
          "minimum": 0,
          "description": "Published random seed"
        },
        "quantile_rule": {
          "type": "string",
          "minLength": 1,
          "description": "Published quantile rule, so that the interval can be recomputed digit for digit"
        },
        "lower": {
          "type": "number"
        },
        "upper": {
          "type": "number"
        }
      },
      "required": [
        "method",
        "level",
        "resamples",
        "seed",
        "quantile_rule",
        "lower",
        "upper"
      ],
      "description": "BCa bootstrap interval over tasks (a task's runs resampled with it) for a task mean such as the task pass rate or the continuous score (section 8.1)."
    },
    "signed_and_absolute": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "signed": {
          "type": "number",
          "minimum": -1,
          "maximum": 1
        },
        "absolute": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        }
      },
      "required": [
        "signed",
        "absolute"
      ],
      "description": "A difference reported with its sign and as an absolute value; thresholds, where set, apply to the absolute value."
    },
    "dispersion": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "geometric_mean": {
          "type": "number",
          "minimum": 0
        },
        "median": {
          "type": "number",
          "minimum": 0
        },
        "q1": {
          "type": "number",
          "minimum": 0
        },
        "q3": {
          "type": "number",
          "minimum": 0
        }
      },
      "required": [
        "geometric_mean",
        "median",
        "q1",
        "q3"
      ],
      "description": "Summary of a ratio to the reference solution: geometric mean, median and quartiles over tasks."
    },
    "error_counts": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "model": {
          "type": "integer",
          "minimum": 0
        },
        "agent": {
          "type": "integer",
          "minimum": 0
        },
        "harness": {
          "type": "integer",
          "minimum": 0
        },
        "application": {
          "type": "integer",
          "minimum": 0
        },
        "provider": {
          "type": "integer",
          "minimum": 0
        },
        "task": {
          "type": "integer",
          "minimum": 0
        },
        "environment": {
          "type": "integer",
          "minimum": 0
        }
      },
      "description": "Run errors by attribution (G7). Subject-side errors are failed runs; initiative-side errors void the task version and are listed with the voided-task list."
    },
    "exam": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "outcome": {
          "type": "string",
          "enum": [
            "passed",
            "not_passed"
          ],
          "description": "passed = every task of the subset passed in all three runs with no veto (section 8.3); any failed run or veto = not_passed"
        },
        "n_tasks": {
          "type": "integer",
          "minimum": 0,
          "description": "Tasks in the subset"
        },
        "claimable_lower_bound": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "With outcome passed: 0.05^(1/n_tasks), the one-sided 95% Clopper–Pearson lower bound with no failure. It bounds the probability that a task of this kind is passed three times in a row by the subject, for tasks built to MedGauge specifications and screened by difficulty class."
        }
      },
      "required": [
        "outcome",
        "n_tasks"
      ],
      "description": "Conclusion layer (exam) for one subset."
    },
    "subset_scores": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "n": {
          "type": "integer",
          "minimum": 0,
          "description": "Tasks in the subset (k = 3 runs each)"
        },
        "n_runs": {
          "type": "integer",
          "minimum": 0,
          "description": "Runs counted; errors count as failed runs and are never removed (G7)"
        },
        "exam": {
          "$ref": "#/$defs/exam"
        },
        "pass_at_3": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M2. Task mean of the unbiased pass@k estimator 1 − C(n−c,k)/C(n,k) with k = 3"
        },
        "pass_pow_3": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M2. Task mean of the unbiased pass^k estimator C(c,k)/C(n,k) with k = 3; the primary ordering key"
        },
        "instability": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M2. pass_at_3 − pass_pow_3"
        },
        "task_pass_rate": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M1. Mean over tasks of the share of passing runs"
        },
        "continuous_score": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M1. Mean over tasks of s = passed checkpoints / m, with s = 0 on a veto; not comparable across families"
        },
        "pass_at_3_ci": {
          "$ref": "#/$defs/clopper_pearson"
        },
        "pass_pow_3_ci": {
          "$ref": "#/$defs/clopper_pearson"
        },
        "task_pass_rate_ci": {
          "$ref": "#/$defs/bca_bootstrap"
        },
        "continuous_score_ci": {
          "$ref": "#/$defs/bca_bootstrap"
        },
        "errors": {
          "$ref": "#/$defs/error_counts"
        },
        "error_rate_subject": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "Subject-side errors / runs"
        },
        "veto_rate": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M6. Runs with any veto / runs"
        },
        "veto_by_category": {
          "type": "object",
          "propertyNames": {
            "$ref": "#/$defs/veto_category"
          },
          "additionalProperties": {
            "type": "integer",
            "minimum": 0
          },
          "description": "M6. Runs with a veto, by category"
        },
        "any_veto": {
          "type": "boolean",
          "description": "True if any run of the subset hit a veto; the row is marked on the board"
        },
        "compliance_rate": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "M6. Passed compliance checkpoints / compliance checkpoints"
        },
        "by_difficulty_class": {
          "type": "object",
          "propertyNames": {
            "enum": [
              "1",
              "2",
              "3",
              "4"
            ]
          },
          "additionalProperties": {
            "type": "object",
            "additionalProperties": false,
            "properties": {
              "n": {
                "type": "integer",
                "minimum": 0
              },
              "task_pass_rate": {
                "type": "number",
                "minimum": 0,
                "maximum": 1
              },
              "pass_pow_3": {
                "type": "number",
                "minimum": 0,
                "maximum": 1
              }
            },
            "required": [
              "n"
            ]
          },
          "description": "M4. Figures stratified by structural difficulty class; classes are ordered, never averaged"
        }
      },
      "required": [
        "n",
        "n_runs",
        "exam",
        "pass_at_3",
        "pass_pow_3",
        "instability",
        "task_pass_rate",
        "continuous_score",
        "errors",
        "veto_rate"
      ]
    },
    "run": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "run": {
          "type": "integer",
          "enum": [
            1,
            2,
            3
          ]
        },
        "pass": {
          "type": "boolean",
          "description": "y_r: every checkpoint passed and no veto"
        },
        "checkpoints_passed": {
          "type": "integer",
          "minimum": 0
        },
        "checkpoints_total": {
          "type": "integer",
          "minimum": 1,
          "description": "m"
        },
        "s": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "passed checkpoints / m; 0 on a veto"
        },
        "veto_categories": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/veto_category"
          },
          "uniqueItems": true
        },
        "error": {
          "oneOf": [
            {
              "type": "null"
            },
            {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "attribution": {
                  "$ref": "#/$defs/error_source"
                },
                "kind": {
                  "type": "string",
                  "enum": [
                    "empty_response",
                    "timeout",
                    "network",
                    "server",
                    "output_error",
                    "forbidden_action",
                    "other"
                  ]
                }
              },
              "required": [
                "attribution",
                "kind"
              ]
            }
          ]
        },
        "actions": {
          "type": "integer",
          "minimum": 0,
          "description": "Action count under the harness version's own action definition (G8)"
        },
        "tool_calls": {
          "type": "integer",
          "minimum": 0
        },
        "tokens_in": {
          "type": "integer",
          "minimum": 0
        },
        "tokens_out": {
          "type": "integer",
          "minimum": 0
        },
        "wall_clock_sec": {
          "type": "number",
          "minimum": 0
        },
        "amount": {
          "type": "number",
          "minimum": 0,
          "description": "Amount under the record's price table version; absent where amounts do not apply"
        }
      },
      "required": [
        "run",
        "pass",
        "checkpoints_passed",
        "checkpoints_total",
        "s",
        "veto_categories",
        "error"
      ]
    },
    "task_outcome": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "task_key": {
          "type": "string",
          "pattern": "^tk-[0-9a-f]{12,64}$",
          "description": "Opaque per-version key; carries no task content and no task label"
        },
        "task_version": {
          "type": "string",
          "pattern": "^\\d+\\.\\d+\\.\\d+$"
        },
        "subset": {
          "type": "string",
          "enum": [
            "dev",
            "holdout",
            "calib"
          ]
        },
        "runs": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/run"
          },
          "minItems": 3,
          "maxItems": 3
        }
      },
      "required": [
        "task_key",
        "task_version",
        "subset",
        "runs"
      ]
    }
  }
}