outlier-detection-and-quality-assessment

Step 1 加载数据并配置环境

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns

# 设置中英文字体以支持可视化显示 (SimHei 或 WenQuanYi)
plt.rcParams['font.sans-serif'] = ['SimHei', 'WenQuanYi Zen Hei', 'DejaVu Sans']
plt.rcParams['axes.unicode_minus'] = False

# 加载数据
file_path = 'data.xlsx'  # 替换为实际文件路径
df = pd.read_excel(file_path)

# 基础信息检查
print(f"数据形状: {df.shape}")
print(f"数据类型:\n{df.dtypes}")
print(df.head())

Step 2 基于 IQR 方法识别异常值

# 自动筛选数值型列进行分析
target_cols = df.select_dtypes(include=[np.number]).columns.tolist()
outlier_summary = []

for col in target_cols:
    data = df[col].dropna()
    if data.empty:
        continue
        
    # 四分位距计算 (IQR)
    Q1 = data.quantile(0.25)
    Q3 = data.quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    
    # 识别异常值
    outliers = data[(data < lower_bound) | (data > upper_bound)]
    
    outlier_summary.append({
        'target_col': col,
        'outlier_count': len(outliers),
        'outlier_ratio': f"{(len(outliers)/len(data)*100):.2f}%",
        'lower_limit': lower_bound,
        'upper_limit': upper_bound,
        'sample_values': outliers.values.tolist()[:5]  # 保留前5个示例
    })

outlier_df = pd.DataFrame(outlier_summary)
print("\n=== 异常值统计汇总 ===")
print(outlier_df.to_string(index=False))

Step 3 生成多维度可视化箱线图

# 配置多子图布局
num_cols = len(target_cols)
cols_per_row = 3
rows = (num_cols + cols_per_row - 1) // cols_per_row

fig, axes = plt.subplots(rows, cols_per_row, figsize=(18, 5 * rows))
fig.suptitle('数据分布与异常值检测箱线图', fontsize=16, fontweight='bold')
axes_flat = axes.flatten()

# 遍历绘制每个维度的分布
for i, col in enumerate(target_cols):
    ax = axes_flat[i]
    # 绘制箱线图并美化
    sns.boxplot(y=df[col].dropna(), ax=ax, color='skyblue', width=0.4,
                flierprops=dict(marker='o', markerfacecolor='red', markersize=5, alpha=0.5))
    
    ax.set_title(f'列: {col}', fontsize=12)
    ax.grid(True, linestyle='--', alpha=0.6)
    
    # 嵌入实时统计标注
    stats = df[col].describe()
    stats_text = f'均值: {stats["mean"]:.2f}\n中位数: {stats["50%"]:.2f}\n标准差: {stats["std"]:.2f}'
    ax.text(0.05, 0.95, stats_text, transform=ax.transAxes, fontsize=9,
            verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))

# 隐藏多余的子图
for j in range(i + 1, len(axes_flat)):
    axes_flat[j].axis('off')

plt.tight_layout(rect=[0, 0.03, 1, 0.95])
output_path = 'outlier_analysis_report.png'
plt.savefig(output_path, dpi=300, bbox_inches='tight')
plt.show()