test-reporting-triage-skill

Test Reporting & Triage Skill

Automatically triage test failures and suggest next actions.

Failure Categorization

// types/test-failure.ts
export type FailureCategory =
  | "timeout"
  | "assertion"
  | "network"
  | "database"
  | "authentication"
  | "permission"
  | "configuration"
  | "flaky"
  | "infrastructure"
  | "unknown";

export interface TestFailure {
  testName: string;
  category: FailureCategory;
  errorMessage: string;
  stackTrace: string;
  suggestedOwner: string;
  suggestedFixes: string[];
  runId: string;
  timestamp: Date;
}

Failure Analyzer

// analyzers/failure-analyzer.ts
export class FailureAnalyzer {
  categorize(error: Error, testName: string): TestFailure {
    const errorMessage = error.message.toLowerCase();
    const stackTrace = error.stack || "";

    // Timeout detection
    if (errorMessage.includes("timeout") || errorMessage.includes("exceeded")) {
      return {
        testName,
        category: "timeout",
        errorMessage: error.message,
        stackTrace,
        suggestedOwner: "Performance Team",
        suggestedFixes: [
          "Check if API is slow",
          "Increase timeout value",
          "Optimize database query",
          "Check for network issues",
        ],
        runId: process.env.CI_RUN_ID || "local",
        timestamp: new Date(),
      };
    }

    // Network errors
    if (
      errorMessage.includes("econnrefused") ||
      errorMessage.includes("network") ||
      errorMessage.includes("fetch failed")
    ) {
      return {
        testName,
        category: "network",
        errorMessage: error.message,
        stackTrace,
        suggestedOwner: "DevOps Team",
        suggestedFixes: [
          "Check if service is running",
          "Verify network connectivity",
          "Check firewall rules",
          "Verify DNS resolution",
        ],
        runId: process.env.CI_RUN_ID || "local",
        timestamp: new Date(),
      };
    }

    // Database errors
    if (
      errorMessage.includes("database") ||
      errorMessage.includes("prisma") ||
      errorMessage.includes("unique constraint")
    ) {
      return {
        testName,
        category: "database",
        errorMessage: error.message,
        stackTrace,
        suggestedOwner: "Backend Team",
        suggestedFixes: [
          "Check database connection",
          "Verify test data cleanup",
          "Check for race conditions",
          "Review migration status",
        ],
        runId: process.env.CI_RUN_ID || "local",
        timestamp: new Date(),
      };
    }

    // Authentication errors
    if (
      errorMessage.includes("unauthorized") ||
      errorMessage.includes("authentication") ||
      errorMessage.includes("401")
    ) {
      return {
        testName,
        category: "authentication",
        errorMessage: error.message,
        stackTrace,
        suggestedOwner: "Auth Team",
        suggestedFixes: [
          "Check auth token validity",
          "Verify test user credentials",
          "Check session expiration",
          "Review auth middleware",
        ],
        runId: process.env.CI_RUN_ID || "local",
        timestamp: new Date(),
      };
    }

    // Assertion failures
    if (
      errorMessage.includes("expected") &&
      errorMessage.includes("received")
    ) {
      return {
        testName,
        category: "assertion",
        errorMessage: error.message,
        stackTrace,
        suggestedOwner: this.determineOwnerFromPath(stackTrace),
        suggestedFixes: [
          "Review recent code changes",
          "Check if test expectations are correct",
          "Verify test data setup",
          "Check for breaking changes",
        ],
        runId: process.env.CI_RUN_ID || "local",
        timestamp: new Date(),
      };
    }

    // Default: unknown
    return {
      testName,
      category: "unknown",
      errorMessage: error.message,
      stackTrace,
      suggestedOwner: "On-Call Engineer",
      suggestedFixes: [
        "Review error message and stack trace",
        "Check recent commits",
        "Run test locally to reproduce",
        "Add more specific error handling",
      ],
      runId: process.env.CI_RUN_ID || "local",
      timestamp: new Date(),
    };
  }

  private determineOwnerFromPath(stackTrace: string): string {
    if (stackTrace.includes("/frontend/")) return "Frontend Team";
    if (stackTrace.includes("/backend/")) return "Backend Team";
    if (stackTrace.includes("/api/")) return "API Team";
    if (stackTrace.includes("/database/")) return "Database Team";
    return "Development Team";
  }
}