observability

Observability Skill

Everything needed to understand what your system is doing in production: structured logging, distributed tracing, metrics, and error tracking.

1. Structured Logging

Core Principles

• Emit JSON that machines can parse, not prose that humans must parse
• Every log line: timestamp, level, message, correlation_id, service
• Never log PII — redact before logging
• Use log levels semantically (DEBUG/INFO/WARNING/ERROR — no print())

Python — structlog or loguru

# structlog (recommended for services)
import structlog
import uuid

logger = structlog.get_logger()

# Configure at app startup
structlog.configure(
    processors=[
        structlog.contextvars.merge_contextvars,
        structlog.processors.add_log_level,
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.processors.JSONRenderer(),
    ]
)

# Bind context for a request/task
structlog.contextvars.bind_contextvars(
    correlation_id=str(uuid.uuid4()),
    user_id=user.id,          # Safe — internal ID, not PII
    service="user-service",
)

logger.info("user.login", method="oauth2", provider="google")
logger.warning("rate.limit.approaching", remaining=5, limit=100)
logger.error("db.query.failed", query="users.find", error=str(e))

# loguru (recommended for simpler scripts/notebooks)
from loguru import logger
import sys

logger.remove()  # Remove default stderr handler
logger.add(
    sys.stderr,
    format="{time:ISO8601} {level} {message}",
    serialize=True,            # JSON output
    level="INFO",
)
logger.add("logs/app.log", rotation="100 MB", retention="7 days", serialize=True)

TypeScript — pino

import pino from 'pino'

export const logger = pino({
  level: process.env.LOG_LEVEL ?? 'info',
  transport: process.env.NODE_ENV === 'development'
    ? { target: 'pino-pretty', options: { colorize: true } }
    : undefined,
  base: { service: 'web-api', env: process.env.NODE_ENV },
})

// Child logger with bound context
export function requestLogger(correlationId: string) {
  return logger.child({ correlationId })
}