csv-transform

CSV Transform Skill

Limpar, validar e transformar CSVs sujos em dados prontos para uso.

Quick Reference

Diagnostico Rapido

Sempre rodar antes de qualquer transformacao:

import pandas as pd
import chardet

filepath = "data.csv"

# 1. Detectar encoding
with open(filepath, 'rb') as f:
    raw = f.read(10000)
    result = chardet.detect(raw)
    print(f"Encoding: {result['encoding']} (confidence: {result['confidence']:.0%})")

# 2. Detectar separador
with open(filepath, 'r', encoding=result['encoding'], errors='replace') as f:
    first_lines = [f.readline() for _ in range(5)]
    for sep_name, sep_char in [('comma', ','), ('semicolon', ';'), ('tab', '\t'), ('pipe', '|')]:
        counts = [line.count(sep_char) for line in first_lines]
        if min(counts) > 0 and max(counts) == min(counts):
            print(f"Separador: {sep_name} ({sep_char!r})")
            break

# 3. Carregar e diagnosticar
df = pd.read_csv(filepath, encoding=result['encoding'], sep=sep_char)
print(f"\nShape: {df.shape}")
print(f"Colunas: {list(df.columns)}")
print(f"\nTipos:\n{df.dtypes}")
print(f"\nNulls:\n{df.isnull().sum()}")
print(f"\nDuplicatas: {df.duplicated().sum()}")
print(f"\nAmostra:\n{df.head()}")

Limpeza de Headers

# Strip whitespace, lowercase, snake_case
import re

def clean_columns(df):
    df.columns = (
        df.columns
        .str.strip()
        .str.lower()
        .str.replace(r'[^\w\s]', '', regex=True)
        .str.replace(r'\s+', '_', regex=True)
        .str.replace(r'_+', '_', regex=True)
        .str.strip('_')
    )
    return df

df = clean_columns(df)

# Renomear colunas especificas
df = df.rename(columns={
    'nome_completo': 'name',
    'data_nascimento': 'birth_date',
    'cpf_cnpj': 'document',
})