prompt-versioning

Prompt Versioning & A/B Testing

Track prompt versions in production and compare their performance.

Core Principle

Every LLM call should be traceable to:

1. Which prompt template was used
2. Which version of that template
3. What variables were injected
4. How it performed (quality, cost, latency)

Without this, you can't iterate on prompts with confidence.

Prompt Version Attributes

# P0 - Always capture
span.set_attribute("prompt.template_id", "researcher_system_v3")
span.set_attribute("prompt.version", "3.2.1")
span.set_attribute("prompt.variant", "control")  # or "treatment_a"

# P1 - For debugging
span.set_attribute("prompt.template_hash", hash(template))
span.set_attribute("prompt.variables", ["task", "context", "tools"])
span.set_attribute("prompt.char_count", len(rendered_prompt))

Basic Version Tracking

from dataclasses import dataclass
from langfuse.decorators import observe, langfuse_context

@dataclass
class PromptVersion:
    template_id: str
    version: str
    template: str

PROMPTS = {
    "researcher_v1": PromptVersion(
        template_id="researcher_system",
        version="1.0.0",
        template="You are a research assistant. {task}",
    ),
    "researcher_v2": PromptVersion(
        template_id="researcher_system",
        version="2.0.0",
        template="""You are an expert research analyst.

Task: {task}

Guidelines:
- Cite sources
- Be concise
- Highlight uncertainties""",
    ),
}

@observe(name="llm.call", as_type="generation")
def call_llm(messages: list, prompt_key: str = "researcher_v2"):
    prompt = PROMPTS[prompt_key]

    langfuse_context.update_current_observation(
        metadata={
            "prompt_template_id": prompt.template_id,
            "prompt_version": prompt.version,
            "prompt_key": prompt_key,
        }
    )

    response = client.messages.create(
        model="claude-3-5-sonnet-latest",
        system=prompt.template.format(**variables),
        messages=messages,
    )

    return response