benchmark-pii-recall

Benchmark PII recall

Measure PII recall before optimizing F1, size, or latency. A missed direct identifier is a privacy failure even when aggregate F1 improves.

Procedure

1. Build synthetic fixtures with exact offsets and canonical PII labels.
2. Include direct identifiers, boundary cases, languages/scripts, and the target device or quantization.
3. Run extract_pii at the candidate threshold.
4. Normalize prediction labels and score each document separately.
5. Aggregate counts only; do not persist raw text or identifier surfaces.
6. Fail the release when the recall floor or zero-critical-leak requirement is not met.

Runnable synthetic benchmark

Install the model runtime first with python -m pip install "openmed[hf]".

from openmed import extract_pii
from openmed.core.labels import normalize_label
from openmed.eval import compute_character_recall, compute_exact_span_f1

MODEL = "OpenMed/OpenMed-PII-SuperClinical-Small-44M-v1"
RECALL_FLOOR = 0.99
FIXTURES = [
    {
        "text": (
            "Call the synthetic clinic at 212-555-0198 or email "
            "[email protected]."
        ),
        "spans": [
            ("PHONE", "212-555-0198"),
            ("EMAIL", "[email protected]"),
        ],
    },
    {
        "text": (
            "The synthetic callback number is 415-555-0136 and the contact "
            "address is [email protected]."
        ),
        "spans": [
            ("PHONE", "415-555-0136"),
            ("EMAIL", "[email protected]"),
        ],
    },
]

true_positives = false_positives = false_negatives = 0
covered_graphemes = total_graphemes = 0

for fixture in FIXTURES:
    text = fixture["text"]
    gold = []
    for label, surface in fixture["spans"]:
        start = text.index(surface)
        gold.append(
            {"start": start, "end": start + len(surface), "label": label}
        )

    result = extract_pii(
        text,
        model_name=MODEL,
        confidence_threshold=0.5,
        lang="en",
    )
    predicted = [
        {
            "start": entity.start,
            "end": entity.end,
            "label": normalize_label(entity.label),
        }
        for entity in result.entities
        if entity.start is not None and entity.end is not None
    ]

    exact = compute_exact_span_f1(gold, predicted, source_text=text)
    recall = compute_character_recall(gold, predicted, source_text=text)
    true_positives += exact.true_positives
    false_positives += exact.false_positives
    false_negatives += exact.false_negatives
    covered_graphemes += int(recall.numerator)
    total_graphemes += int(recall.denominator)

exact_recall = true_positives / max(true_positives + false_negatives, 1)
grapheme_recall = covered_graphemes / max(total_graphemes, 1)
print(
    {
        "documents": len(FIXTURES),
        "exact_span_recall": exact_recall,
        "grapheme_recall": grapheme_recall,
        "false_positives": false_positives,
        "false_negatives": false_negatives,
    }
)
assert grapheme_recall >= RECALL_FLOOR, "PII recall floor not met"