experiment-tracking-and-reproducibility

Experiment Tracking and Reproducibility

Overview

A metric nobody can reproduce is an anecdote. Reproducing a run needs five things: the code (git SHA), the data (a version or hash), the parameters, the environment (lockfile) and the seed. Log all five with every run, and make the training entrypoint a deterministic function of them.

Core principle: if two runs with the same five inputs can produce different models, the difference must be explainable (GPU non-determinism, thread scheduling) and bounded — never "randomness" from an unseeded call.

1. What every run logs

def data_fingerprint(df: pd.DataFrame) -> str:
    return hashlib.sha256(pd.util.hash_pandas_object(df, index=True).to_numpy().tobytes()).hexdigest()[:16]

def code_and_env() -> dict[str, str]:
    sha = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip()
    dirty = bool(subprocess.run(["git", "status", "--porcelain"], capture_output=True, text=True, check=True).stdout)
    lock = hashlib.sha256(Path("uv.lock").read_bytes()).hexdigest()[:16]
    return {"git_sha": sha, "git_dirty": str(dirty), "lock_sha": lock, "python": platform.python_version()}

2. MLflow 3

with mlflow.start_run(run_name=f"churn-{cfg.seed}"):
    mlflow.log_params(cfg.model_dump())
    mlflow.set_tags({**code_and_env(), "data_fingerprint": data_fingerprint(train_df), "seed": str(cfg.seed)})
    mlflow.log_input(mlflow.data.from_pandas(train_df, name="train", targets="churned"), context="training")

    cv = cross_validate(pipe, X_train, y_train, cv=splitter, scoring="average_precision")
    mlflow.log_metrics({
        "cv_ap_mean": cv["test_score"].mean(),
        "cv_ap_std": cv["test_score"].std(),
        "baseline_ap": y_train.mean(),
    })

    pipe.fit(X_train, y_train)
    info = mlflow.sklearn.log_model(
        sk_model=pipe,
        name="model",
        input_example=X_train.head(5),
        skops_trusted_types=REVIEWED_SKLEARN_TYPES,
    )