verification-before-delivery

Verification Before Delivery

The final gate before delivering any analytical output. No delivery is approved without passing all three verification stages: artifact integrity, statistical evidence audit, and reproducibility confirmation.

Why this gate exists: "It runs on my machine" is not a delivery standard. Analyses fail in production due to missing artifacts, unreproducible seeds, undocumented assumptions, and missing confidence intervals. This skill catches those failures before they reach stakeholders.

Iron Law

NO DELIVERY WITHOUT REPRODUCIBLE EVIDENCE. CONFIDENCE INTERVALS ARE MANDATORY ON ALL REPORTED METRICS.

When to Use

Trigger this skill when:

• User says "done", "complete", "ready to deliver", "finished"
• A PR is being prepared for analysis code
• A report is being sent to a stakeholder
• A model artifact is being moved to staging or production

Step-by-Step Procedure

Stage 1 — Artifact Integrity Check

Verify every expected output exists, is non-empty, and is loadable.

import os
import json
import joblib
import pandas as pd
from pathlib import Path

def verify_artifact_integrity(manifest_path: str = "artifacts/analysis_manifest.json") -> dict:
    """
    Verify all artifacts referenced in the manifest actually exist and are loadable.
    Returns a dict with pass/fail status per artifact.
    """
    manifest = json.loads(Path(manifest_path).read_text())
    results = {}

    # Check manifest itself is valid
    assert manifest.get("project"), "manifest.project is empty"
    assert manifest.get("brainstorming", {}).get("primary_metric"), \
        "FAIL: primary_metric not declared in brainstorming"

    # Check each stage's artifact paths
    artifact_fields = {
        "data_profiling": "profile_path",
        "data_exploration": "eda_report_path",
        "data_validation": "tdds_report_path",
        "leakage_guard": "report_path",
        "feature_engineering": "registry_path",
        "model_evaluation": "shap_path",
        "report": "report_path",
    }

    for stage, field in artifact_fields.items():
        path = manifest.get(stage, {}).get(field)
        if path is None:
            results[f"{stage}.{field}"] = "SKIP (null)"
            continue
        if not os.path.exists(path):
            results[f"{stage}.{field}"] = f"FAIL: file not found at {path}"
        elif os.path.getsize(path) == 0:
            results[f"{stage}.{field}"] = f"FAIL: file is empty at {path}"
        else:
            results[f"{stage}.{field}"] = f"PASS ({os.path.getsize(path):,} bytes)"

    # Check transformer artifacts are loadable
    for pkl_path in manifest.get("feature_engineering", {}).get("transformer_paths", []):
        try:
            obj = joblib.load(pkl_path)
            results[f"transformer:{pkl_path}"] = f"PASS (type: {type(obj).__name__})"
        except Exception as e:
            results[f"transformer:{pkl_path}"] = f"FAIL: cannot load — {e}"

    # Check model artifact
    model_eval = manifest.get("model_evaluation", {})
    if model_eval.get("shap_path"):
        if not os.path.exists(model_eval["shap_path"]):
            results["shap_plot"] = "FAIL: SHAP plot missing"

    return results

results = verify_artifact_integrity()
failures = {k: v for k, v in results.items() if v.startswith("FAIL")}
if failures:
    print("❌ ARTIFACT INTEGRITY FAILURES:")
    for k, v in failures.items():
        print(f"  {k}: {v}")
    raise RuntimeError("Delivery blocked: artifact integrity failures")
else:
    print("✅ All artifact integrity checks passed")
    for k, v in results.items():
        print(f"  {k}: {v}")