prometheus-alertmanager

Skill: Prometheus & Alertmanager

Expertise: PromQL, alert rules, recording rules, Alertmanager routing, inhibition, silences.

When to load

When writing alert rules, debugging PromQL, configuring Alertmanager routing, or investigating a firing alert.

Golden Signal Alert Rules

# alerts/service-golden-signals.yaml
groups:
  - name: service.golden-signals
    rules:

      # ── Errors ────────────────────────────────────────
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[5m])) by (namespace, service)
            /
            sum(rate(http_requests_total[5m])) by (namespace, service)
          ) > 0.01
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Error rate > 1% — {{ $labels.service }} in {{ $labels.namespace }}"
          description: "Current error rate: {{ $value | humanizePercentage }}"
          runbook_url: "https://runbooks.internal/high-error-rate"

      # ── Latency ───────────────────────────────────────
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (namespace, service, le)
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "p99 latency > 1s — {{ $labels.service }}"
          description: "p99: {{ $value | humanizeDuration }}"
          runbook_url: "https://runbooks.internal/high-latency"

      # ── Saturation ────────────────────────────────────
      - alert: PodMemoryPressure
        expr: |
          (
            container_memory_working_set_bytes{container!=""}
            /
            container_spec_memory_limit_bytes{container!=""}
          ) > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Memory > 85% limit — {{ $labels.container }} in {{ $labels.namespace }}"
          runbook_url: "https://runbooks.internal/memory-pressure"

      # ── Traffic Drop ──────────────────────────────────
      - alert: TrafficDrop
        expr: |
          (
            sum(rate(http_requests_total[5m])) by (service)
            /
            sum(rate(http_requests_total[1h] offset 5m)) by (service)
          ) < 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Traffic dropped > 50% vs 1h ago — {{ $labels.service }}"
          runbook_url: "https://runbooks.internal/traffic-drop"