{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://medgauge.org/schemas/0.1.0/benchmark.schema.json",
  "title": "benchmark",
  "description": "A benchmark card (benchmark registry): scenario, capabilities served, task form, scoring, integrity controls, frozen versions and the public/closed boundary.",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "id": {
      "type": "string",
      "pattern": "^bm-[a-z0-9-]+$"
    },
    "name_zh": {
      "type": "string",
      "minLength": 1
    },
    "name_en": {
      "type": "string",
      "minLength": 1
    },
    "status": {
      "type": "string",
      "enum": [
        "active",
        "proposed",
        "experimental",
        "deprecated",
        "retired",
        "planned"
      ]
    },
    "withheld": {
      "type": "object",
      "description": "Card kept in the source tree but not published while it is under review: no page, no board, no evidence; listed by name only.",
      "properties": {
        "since": {
          "type": "string",
          "format": "date"
        },
        "reason_en": {
          "type": "string",
          "minLength": 1
        }
      },
      "required": [
        "since",
        "reason_en"
      ],
      "additionalProperties": false
    },
    "versions": {
      "type": "array",
      "minItems": 0,
      "items": {
        "type": "object",
        "properties": {
          "version": {
            "type": "string",
            "pattern": "^\\d+\\.\\d+\\.\\d+$"
          },
          "frozen_at": {
            "type": "string",
            "format": "date"
          },
          "n_tasks": {
            "type": "object",
            "properties": {
              "dev": {
                "type": "integer",
                "minimum": 0
              },
              "holdout": {
                "type": "integer",
                "minimum": 0
              },
              "calib": {
                "type": "integer",
                "minimum": 0
              }
            },
            "required": [
              "dev",
              "holdout",
              "calib"
            ],
            "additionalProperties": false
          },
          "hashes_url": {
            "type": "string",
            "minLength": 1
          },
          "changelog": {
            "type": "string",
            "minLength": 1
          },
          "hashes_status": {
            "type": "string",
            "enum": [
              "published",
              "pending"
            ],
            "description": "pending = hash manifest not yet released on this site"
          }
        },
        "required": [
          "version",
          "frozen_at",
          "n_tasks",
          "changelog"
        ],
        "additionalProperties": false
      }
    },
    "scenario_zh": {
      "type": "string",
      "minLength": 1
    },
    "scenario_en": {
      "type": "string"
    },
    "task_form": {
      "type": "string",
      "minLength": 1
    },
    "materials": {
      "type": "array",
      "items": {
        "type": "string",
        "minLength": 1
      }
    },
    "scoring": {
      "type": "object",
      "properties": {
        "form": {
          "type": "string",
          "enum": [
            "program",
            "checkpoint",
            "rubric"
          ]
        },
        "pass_rule": {
          "type": "string",
          "minLength": 1
        },
        "veto": {
          "type": "array",
          "items": {
            "type": "string",
            "pattern": "^[ABCDEX]\\.\\d+(\\.\\d+)?$"
          }
        },
        "stability": {
          "type": "string",
          "minLength": 1
        },
        "metrics_reported": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        },
        "assertion_veto": {
          "type": "boolean",
          "description": "true = a violated veto-flagged grading reference linked to a task fails that task outright, and the hit is counted under the X-domain capability the reference maps to"
        }
      },
      "required": [
        "form",
        "pass_rule",
        "veto",
        "stability"
      ],
      "additionalProperties": false
    },
    "difficulty_protocol": {
      "type": "string",
      "minLength": 1
    },
    "integrity": {
      "type": "object",
      "properties": {
        "two_gates": {
          "type": "object",
          "properties": {
            "oracle_full_score": {
              "type": "boolean"
            },
            "five_bad_artifacts_blocked": {
              "type": "boolean"
            },
            "record": {
              "type": "string",
              "minLength": 1,
              "description": "Reference to the gate run record in the private grading repository (package version and summary id). Shown on the card only when present."
            }
          },
          "required": [
            "oracle_full_score",
            "five_bad_artifacts_blocked"
          ],
          "additionalProperties": false
        },
        "expert_key_card": {
          "type": "boolean"
        },
        "independent_review": {
          "type": "boolean"
        },
        "contamination_control": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        },
        "retirement_rule": {
          "type": "string",
          "minLength": 1
        },
        "records": {
          "type": "object",
          "additionalProperties": false,
          "description": "References to the evidence held in the private grading repository. A field is displayed on the card only when its record reference is present.",
          "properties": {
            "expert_key_cards": {
              "type": "string",
              "minLength": 1
            },
            "independent_review": {
              "type": "string",
              "minLength": 1
            },
            "difficulty_prediction": {
              "type": "string",
              "minLength": 1
            }
          }
        }
      },
      "required": [
        "two_gates",
        "expert_key_card",
        "independent_review",
        "contamination_control",
        "retirement_rule"
      ],
      "additionalProperties": false
    },
    "deidentification": {
      "type": "string",
      "minLength": 1
    },
    "external_anchors": {
      "type": "array",
      "items": {
        "type": "string",
        "minLength": 1
      }
    },
    "access": {
      "type": "object",
      "properties": {
        "public": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        },
        "closed": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        },
        "how_to_test": {
          "type": "string",
          "minLength": 1
        }
      },
      "required": [
        "public",
        "closed",
        "how_to_test"
      ],
      "additionalProperties": false
    },
    "predictions": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "model_tier": {
            "type": "string",
            "minLength": 1
          },
          "predicted_pass": {
            "type": "number",
            "minimum": 0,
            "maximum": 1
          },
          "basis": {
            "type": "string"
          }
        },
        "required": [
          "model_tier",
          "predicted_pass"
        ],
        "additionalProperties": false
      }
    },
    "citations": {
      "type": "array",
      "items": {
        "type": "string",
        "minLength": 1
      }
    },
    "capabilities": {
      "type": "array",
      "minItems": 1,
      "items": {
        "type": "string",
        "pattern": "^[ABCDEX]\\.\\d+(\\.\\d+)?$"
      }
    },
    "source_groups": {
      "type": "array",
      "minItems": 1,
      "items": {
        "type": "string",
        "minLength": 1
      },
      "description": "Source groups: directories within the family grouped by source of material; never a unit of score, results are reported per family"
    },
    "difficulty_line": {
      "type": "object",
      "properties": {
        "rule": {
          "type": "string",
          "minLength": 1
        },
        "as_of": {
          "type": "string",
          "format": "date"
        },
        "n_formal": {
          "type": "integer",
          "minimum": 0
        },
        "n_at_line": {
          "type": "integer",
          "minimum": 0
        }
      },
      "required": [
        "rule",
        "as_of"
      ],
      "additionalProperties": false
    },
    "task_types": {
      "type": "array",
      "items": {
        "type": "string",
        "enum": [
          "detect_and_log",
          "item_by_item_judgement",
          "structured_answer",
          "transform_output",
          "multi_artifact_checkpoint"
        ]
      }
    },
    "family_notes": {
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "family": {
            "type": "string",
            "minLength": 1
          },
          "label": {
            "type": "string",
            "minLength": 1
          },
          "note": {
            "type": "string",
            "minLength": 1
          }
        },
        "required": [
          "family",
          "label",
          "note"
        ]
      },
      "description": "Notes on specific task families or source groups filed under this card, e.g. cross-domain families"
    },
    "artifact_fields": {
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "name": {
            "type": "string"
          },
          "type": {
            "type": "string"
          },
          "meaning": {
            "type": "string"
          }
        },
        "required": [
          "name",
          "type",
          "meaning"
        ]
      }
    },
    "defect_classes": {
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "id": {
            "type": "string",
            "pattern": "^[a-z0-9_]+$"
          },
          "name": {
            "type": "string"
          },
          "description": {
            "type": "string"
          }
        },
        "required": [
          "id",
          "name",
          "description"
        ]
      },
      "description": "Classes of defect or verdict the family covers, at method level; not one-to-one with tasks"
    },
    "scoring_details": {
      "type": "array",
      "items": {
        "type": "string",
        "minLength": 1
      }
    },
    "illustration": {
      "type": "object",
      "additionalProperties": false,
      "description": "Synthetic illustration of the task shape. Not taken from any task.",
      "properties": {
        "input": {
          "type": "string"
        },
        "artifact": {
          "type": "string"
        },
        "note": {
          "type": "string"
        }
      },
      "required": [
        "input",
        "artifact",
        "note"
      ]
    },
    "plan": {
      "type": "object",
      "additionalProperties": false,
      "description": "Roadmap state for a planned benchmark. Nothing here is a result.",
      "properties": {
        "stage": {
          "type": "string",
          "enum": [
            "design",
            "item_building",
            "calibration"
          ]
        },
        "added": {
          "type": "string",
          "format": "date"
        },
        "rationale": {
          "type": "string"
        },
        "next_steps": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        },
        "open_questions": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          }
        }
      },
      "required": [
        "stage",
        "added",
        "next_steps"
      ]
    }
  },
  "required": [
    "id",
    "name_en",
    "capabilities",
    "status",
    "versions",
    "scenario_en",
    "task_form",
    "materials",
    "scoring",
    "difficulty_protocol",
    "integrity",
    "deidentification",
    "access",
    "source_groups"
  ]
}