domino-trace-setup

/domino-trace-setup Command

Set up GenAI tracing for agent and LLM applications in Domino.

Usage

/domino-trace-setup

What This Command Does

1. Checks environment requirements (MLflow 3.2.0, Domino SDK)
2. Creates tracing setup module with decorators and context managers
3. Generates example evaluators for quality scoring
4. Creates config.yaml for agent configuration
5. Provides example traced agent code

Output Files

tracing_setup.py

"""
Domino GenAI Tracing Setup
Generated by /domino-trace-setup

Requirements:
- mlflow==3.2.0
- dominodatalab[data,aisystems] @ git+https://github.com/dominodatalab/python-domino.git@master
"""

import mlflow
from domino.agents.tracing import add_tracing
from domino.agents.logging import DominoRun
import os

def setup_tracing(framework: str = "openai"):
    """
    Enable auto-tracing for LLM framework.

    Args:
        framework: One of 'openai', 'anthropic', 'langchain'
    """
    if framework == "openai":
        mlflow.openai.autolog()
    elif framework == "anthropic":
        mlflow.anthropic.autolog()
    elif framework == "langchain":
        mlflow.langchain.autolog()
    else:
        raise ValueError(f"Unknown framework: {framework}")

    print(f"Enabled {framework} auto-tracing")

def create_evaluator(metrics: list = None):
    """
    Create a basic evaluator function.

    Args:
        metrics: List of metrics to evaluate

    Returns:
        Evaluator function for @add_tracing
    """
    if metrics is None:
        metrics = ["quality_score", "response_length"]

    def evaluator(inputs, output):
        """
        Evaluate agent output.

        Args:
            inputs: Dict of function arguments
            output: Function return value

        Returns:
            Dict of metric names to values
        """
        scores = {}

        # Response length
        if isinstance(output, str):
            scores["response_length"] = len(output)
        elif isinstance(output, dict):
            scores["response_length"] = len(str(output))

        # Placeholder for quality score
        # Replace with actual evaluation logic
        scores["quality_score"] = 0.8

        return scores

    return evaluator

# Default evaluator
default_evaluator = create_evaluator()