evals

{
  "name": "evals",
  "version": "0.10.0",
  "description": "LLM evaluation methodology and eval-suite design, based on Anthropic's evaluation guidance. /evals:methodology answers questions on success criteria, eval design, and grading. /evals:design interviews for measurable success criteria and scaffolds a graded eval suite for an LLM app or Claude Code skill. /evals:plugin-eval preflights the CLI and target, prices a suite before running, and reads the with-versus-without delta. /evals:validate checks case files with no model call.",
  "author": {
    "name": "Melodic Software",
    "email": "[email protected]"
  },
  "license": "MIT",
  "userConfig": {
    "max_cost_usd": {
      "type": "number",
      "title": "Eval run cost ceiling (USD)",
      "description": "Ceiling /evals:plugin-eval passes to the CLI as --max-cost-usd. It bounds one invocation, not a session's total, and a run that reaches it stops mid-suite with partial results. Raise it for a suite whose estimate exceeds it.",
      "default": 5,
      "min": 0
    },
    "unlimited_cost": {
      "type": "boolean",
      "title": "No cost ceiling",
      "description": "Drop --max-cost-usd from the invocation so a suite always runs to completion. The estimate is still printed before the run; only the ceiling goes away.",
      "default": false
    },
    "split_policy": {
      "type": "string",
      "title": "Train/test split policy",
      "description": "How eval cases split when a skill or prompt is tuned against them. train-test (the default) splits train and test as the bundled hillclimb guide does; reporting-only also holds back a split that no keep-or-revert decision or final pick reads, used only to report the result.",
      "default": "train-test",
      "options": ["train-test", "reporting-only"]
    },
    "interval_method": {
      "type": "string",
      "title": "Interval method for pass counts",
      "description": "Interval the noise report puts on the count of cases that p