{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://theamateur.co.uk/schemas/dpl/dpl.bench-1.schema.json",
  "title": "DPL readiness benchmark (dpl.bench/1)",
  "description": "What `dpl bench <corpus-dir> --format json` writes: how well the edit-readiness rules find the faults planted in a corpus of plans, scoring every mutant against its own plan. Only open findings that the mutant has and its plan does not (matched by finding id) count. Every member listed here is always written, or left out when null; nothing else appears.",
  "$comment": "A planted fault (a mutant with expected codes) has outcome `caught`, `caught_other`, `masked` or `missed`; a decoy (a correct change, no expected codes) has `alarm` or `quiet`. `clean_plans` lists the open findings on the unmutated plans: they are not called false positives, because that needs a human. `dpl bench` exits 0 when every fault is `caught` and no decoy is an `alarm`, 2 otherwise.",
  "type": "object",
  "properties": {
    "summary": {
      "$ref": "#/$defs/BenchSummary"
    },
    "mutants": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/MutationResult"
      }
    },
    "clean_plans": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/CleanResult"
      }
    },
    "format": {
      "type": "string",
      "const": "dpl.bench/1"
    },
    "rules_version": {
      "type": "string"
    }
  },
  "required": [
    "summary",
    "mutants",
    "clean_plans",
    "format",
    "rules_version"
  ],
  "additionalProperties": false,
  "$defs": {
    "BenchFinding": {
      "type": "object",
      "description": "An open finding that the mutant has and its fixture does not (matched by finding id).",
      "properties": {
        "id": {
          "type": "string",
          "pattern": "^f-[0-9a-f]{12}$"
        },
        "code": {
          "type": "string",
          "pattern": "^(DPL|RDY|REC)-[A-Z0-9-]+$"
        },
        "severity": {
          "$ref": "#/$defs/FindingSeverity"
        },
        "scene": {
          "type": [
            "string",
            "null"
          ]
        },
        "shot": {
          "type": [
            "string",
            "null"
          ]
        },
        "message": {
          "type": "string"
        }
      },
      "required": [
        "id",
        "code",
        "severity",
        "message"
      ],
      "additionalProperties": false
    },
    "BenchMasking": {
      "type": "object",
      "description": "The fixture's own open finding that already carried the expected code at a place the mutation touches.",
      "properties": {
        "finding_id": {
          "type": "string",
          "pattern": "^f-[0-9a-f]{12}$"
        },
        "code": {
          "type": "string",
          "pattern": "^(DPL|RDY|REC)-[A-Z0-9-]+$"
        },
        "evidence_path": {
          "type": "string"
        }
      },
      "required": [
        "finding_id",
        "code",
        "evidence_path"
      ],
      "additionalProperties": false
    },
    "BenchOutcome": {
      "type": "string",
      "description": "What became of one mutant when scored against its own fixture.",
      "enum": [
        "caught",
        "caught_other",
        "masked",
        "missed",
        "alarm",
        "quiet"
      ]
    },
    "BenchSummary": {
      "type": "object",
      "properties": {
        "faults": {
          "type": "integer",
          "minimum": 0
        },
        "caught": {
          "type": "integer",
          "minimum": 0
        },
        "caught_other": {
          "type": "integer",
          "minimum": 0
        },
        "masked": {
          "type": "integer",
          "minimum": 0
        },
        "missed": {
          "type": "integer",
          "minimum": 0
        },
        "recall": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        },
        "recall_or_other": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        },
        "faults_with_extra_new": {
          "type": "integer",
          "minimum": 0
        },
        "safe_fix_offered": {
          "type": "integer",
          "minimum": 0
        },
        "safe_fix_clears": {
          "type": "integer",
          "minimum": 0
        },
        "decoys": {
          "type": "integer",
          "minimum": 0
        },
        "decoy_alarms": {
          "type": "integer",
          "minimum": 0
        },
        "clean_plans": {
          "type": "integer",
          "minimum": 0
        },
        "clean_plans_with_findings": {
          "type": "integer",
          "minimum": 0
        },
        "clean_findings": {
          "type": "integer",
          "minimum": 0
        },
        "clean_findings_by_code": {
          "type": "object",
          "additionalProperties": {
            "type": "integer"
          }
        }
      },
      "required": [
        "faults",
        "caught",
        "caught_other",
        "masked",
        "missed",
        "recall",
        "recall_or_other",
        "faults_with_extra_new",
        "safe_fix_offered",
        "safe_fix_clears",
        "decoys",
        "decoy_alarms",
        "clean_plans",
        "clean_plans_with_findings",
        "clean_findings",
        "clean_findings_by_code"
      ],
      "additionalProperties": false
    },
    "CleanFinding": {
      "type": "object",
      "properties": {
        "id": {
          "type": "string",
          "pattern": "^f-[0-9a-f]{12}$"
        },
        "code": {
          "type": "string",
          "pattern": "^(DPL|RDY|REC)-[A-Z0-9-]+$"
        },
        "severity": {
          "$ref": "#/$defs/FindingSeverity"
        },
        "scene": {
          "type": [
            "string",
            "null"
          ]
        },
        "shot": {
          "type": [
            "string",
            "null"
          ]
        },
        "message": {
          "type": "string"
        }
      },
      "required": [
        "id",
        "code",
        "severity",
        "message"
      ],
      "additionalProperties": false
    },
    "CleanResult": {
      "type": "object",
      "description": "An unmutated plan and the open findings it carries. Whether they are false alarms needs a human.",
      "properties": {
        "fixture": {
          "type": "string"
        },
        "findings": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/CleanFinding"
          }
        },
        "not_assessed_scenes": {
          "type": "integer",
          "minimum": 0
        }
      },
      "required": [
        "fixture",
        "findings",
        "not_assessed_scenes"
      ],
      "additionalProperties": false
    },
    "FindingSeverity": {
      "type": "string",
      "enum": [
        "blocker",
        "major",
        "minor",
        "note"
      ]
    },
    "MutationResult": {
      "type": "object",
      "properties": {
        "id": {
          "type": "string"
        },
        "fixture": {
          "type": "string"
        },
        "expect": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "outcome": {
          "$ref": "#/$defs/BenchOutcome"
        },
        "new_findings": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/BenchFinding"
          }
        },
        "other_codes": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "masked_by": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/BenchMasking"
          }
        },
        "extra_new": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/BenchFinding"
          }
        },
        "safe_fix_offered": {
          "type": "boolean"
        },
        "safe_fix_clears": {
          "type": "boolean"
        }
      },
      "required": [
        "id",
        "fixture",
        "expect",
        "outcome",
        "new_findings",
        "other_codes",
        "masked_by",
        "extra_new",
        "safe_fix_offered",
        "safe_fix_clears"
      ],
      "additionalProperties": false
    }
  }
}