text-mining-guide

Text Mining Guide

A skill for applying natural language processing (NLP) and text mining techniques to research data. Covers text preprocessing, feature extraction, topic modeling, sentiment analysis, and named entity recognition for analyzing surveys, abstracts, social media, and document corpora.

Text Preprocessing Pipeline

Standard Cleaning Steps

import re
from collections import Counter


def preprocess_text(text: str, lowercase: bool = True,
                    remove_numbers: bool = False,
                    min_word_length: int = 2) -> list[str]:
    """
    Preprocess text for NLP analysis.

    Args:
        text: Raw input text
        lowercase: Convert to lowercase
        remove_numbers: Remove numeric tokens
        min_word_length: Minimum token length to keep
    """
    if lowercase:
        text = text.lower()

    # Remove URLs
    text = re.sub(r"http\S+|www\.\S+", "", text)

    # Remove HTML tags
    text = re.sub(r"<[^>]+>", "", text)

    # Remove special characters (keep apostrophes for contractions)
    text = re.sub(r"[^a-zA-Z0-9\s']", " ", text)

    # Tokenize
    tokens = text.split()

    if remove_numbers:
        tokens = [t for t in tokens if not t.isdigit()]

    # Remove short tokens
    tokens = [t for t in tokens if len(t) >= min_word_length]

    return tokens


def remove_stopwords(tokens: list[str],
                     custom_stopwords: list[str] = None) -> list[str]:
    """
    Remove stopwords from token list.
    """
    # Minimal English stopwords (extend as needed)
    default_stops = {
        "the", "a", "an", "and", "or", "but", "in", "on", "at",
        "to", "for", "of", "with", "by", "is", "was", "are", "were",
        "be", "been", "being", "have", "has", "had", "do", "does",
        "did", "will", "would", "could", "should", "may", "might",
        "this", "that", "these", "those", "it", "its", "not", "no"
    }

    if custom_stopwords:
        default_stops.update(custom_stopwords)

    return [t for t in tokens if t not in default_stops]