mcp-testing

MCP Eval Testing Patterns

This skill loads eval testing patterns for MCP tools using @umbraco-cms/mcp-server-sdk/evals. Eval tests verify tools work correctly when driven by an LLM agent.

For integration tests, use /build-tools-tests instead.

When to Use

Use this skill when:

• Creating eval/acceptance tests for MCP tools
• Verifying tools work correctly in LLM-driven workflows
• Debugging eval test failures

Setup

Eval tests live in tests/evals/ with a dedicated Jest config. The setup file is loaded automatically via setupFilesAfterEnv — test files do NOT need to import it.

tests/evals/helpers/e2e-setup.ts

import path from "path";
import { configureEvals, ClaudeModels } from "@umbraco-cms/mcp-server-sdk/evals";

configureEvals({
  mcpServerPath: path.resolve(process.cwd(), "dist/index.js"),
  mcpServerName: "my-mcp-server",
  serverEnv: { USE_MOCK_API: "true" },
  defaultModel: ClaudeModels.Haiku,
  defaultMaxTurns: 10,
  defaultMaxBudgetUsd: 0.25,
  defaultTimeoutMs: 60000,
});

tests/evals/jest.config.ts

import type { Config } from "jest";

const config: Config = {
  preset: "ts-jest/presets/js-with-ts-esm",
  testEnvironment: "node",
  extensionsToTreatAsEsm: [".ts"],
  rootDir: "../..",
  testMatch: ["<rootDir>/tests/evals/**/*.test.ts"],
  setupFilesAfterEnv: ["<rootDir>/tests/evals/helpers/e2e-setup.ts"],
  maxConcurrency: 1,
  maxWorkers: 1,
  testTimeout: 120000,
};

export default config;

Test Pattern

// tests/evals/entity-crud.test.ts
import { describe, it } from "@jest/globals";
import {
  runScenarioTest,
  setupConsoleMock,
  getDefaultTimeoutMs,
} from "@umbraco-cms/mcp-server-sdk/evals";

describe("entity evals", () => {
  setupConsoleMock();

  it(
    "should complete workflow",
    runScenarioTest({
      prompt: `Complete these tasks:
1. Create an item named "Test"
2. Delete the item
3. Say "Workflow completed"`,
      tools: ["create-item", "delete-item"],
      requiredTools: ["create-item", "delete-item"],
      successPattern: "Workflow completed",
    }),
    getDefaultTimeoutMs()
  );
});