statistical-analysis

Statistical Analysis Skill

Perform rigorous statistical analysis on SWARM sweep or multi-seed data, including hypothesis tests, effect sizes, and multiple-comparison corrections.

Prerequisites

• scipy>=1.10 for statistical tests
• pandas>=2.0 for data manipulation
• numpy>=1.24 for numerical computation

Procedure

1. Load and normalize data

import pandas as pd

df = pd.read_csv(csv_path)

# Normalize column aliases
alias_map = {
    "tax_rate": "transaction_tax_rate",
    "tax": "transaction_tax_rate",
    "tox": "toxicity_rate",
    "toxicity": "toxicity_rate",
}
df.rename(columns={k: v for k, v in alias_map.items() if k in df.columns}, inplace=True)

2. Pairwise Welch's t-tests

from scipy import stats
from itertools import combinations

param_col = "transaction_tax_rate"  # or detect automatically
metric = "welfare"

groups = {val: grp[metric].values for val, grp in df.groupby(param_col)}
pairs = list(combinations(sorted(groups.keys()), 2))

results = []
for a, b in pairs:
    t_stat, p_val = stats.ttest_ind(groups[a], groups[b], equal_var=False)
    results.append({
        "group_a": float(a),
        "group_b": float(b),
        "t_statistic": float(t_stat),
        "p_value": float(p_val),
    })

3. Effect sizes (Cohen's d)

import numpy as np

def cohens_d(x, y):
    nx, ny = len(x), len(y)
    pooled_std = np.sqrt(((nx-1)*np.std(x,ddof=1)**2 + (ny-1)*np.std(y,ddof=1)**2) / (nx+ny-2))
    if pooled_std == 0:
        return 0.0
    return (np.mean(x) - np.mean(y)) / pooled_std

for r, (a, b) in zip(results, pairs):
    r["cohens_d"] = float(cohens_d(groups[a], groups[b]))
    r["effect_magnitude"] = (
        "large" if abs(r["cohens_d"]) >= 0.8 else
        "medium" if abs(r["cohens_d"]) >= 0.5 else
        "small" if abs(r["cohens_d"]) >= 0.2 else
        "negligible"
    )

4. Bonferroni correction

n_tests = len(results)
bonferroni_threshold = 0.05 / n_tests

for r in results:
    r["bonferroni_significant"] = r["p_value"] < bonferroni_threshold