llm-debugger

LLM Debugger

Systematically diagnose and fix LLM output issues.

Failure Taxonomy

class FailureType(Enum):
    HALLUCINATION = "hallucination"
    FORMAT_VIOLATION = "format_violation"
    CONSTRAINT_BREAK = "constraint_break"
    REASONING_ERROR = "reasoning_error"
    TOOL_MISUSE = "tool_misuse"
    REFUSAL = "unexpected_refusal"
    INCOMPLETE = "incomplete_output"

Root Cause Analysis

def diagnose_failure(input: str, output: str, expected: dict) -> dict:
    """Identify why LLM output failed"""

    issues = []

    # Check format
    if expected.get("format") == "json":
        try:
            json.loads(output)
        except:
            issues.append({
                "type": FailureType.FORMAT_VIOLATION,
                "details": "Invalid JSON output"
            })

    # Check required fields
    if expected.get("required_fields"):
        for field in expected["required_fields"]:
            if field not in output:
                issues.append({
                    "type": FailureType.INCOMPLETE,
                    "details": f"Missing required field: {field}"
                })

    # Check constraints
    if expected.get("max_length"):
        if len(output) > expected["max_length"]:
            issues.append({
                "type": FailureType.CONSTRAINT_BREAK,
                "details": f"Output too long: {len(output)} > {expected['max_length']}"
            })

    # Check for hallucination indicators
    if contains_hallucination_markers(output):
        issues.append({
            "type": FailureType.HALLUCINATION,
            "details": "Contains fabricated information"
        })

    return {
        "has_issues": len(issues) > 0,
        "issues": issues,
        "primary_issue": issues[0] if issues else None
    }

def contains_hallucination_markers(output: str) -> bool:
    """Detect common hallucination patterns"""
    markers = [
        r'According to.*that doesn\'t exist',
        r'In \d{4}.*before that year',
        r'contradicts itself',
    ]
    return any(re.search(marker, output, re.IGNORECASE) for marker in markers)