tidytext

tidytext

Tidy text mining.

Tokenization

library(tidytext)
library(dplyr)

# Tokenize to words
df %>% unnest_tokens(word, text)

# Tokenize to sentences
df %>% unnest_tokens(sentence, text, token = "sentences")

# Tokenize to ngrams
df %>% unnest_tokens(bigram, text, token = "ngrams", n = 2)
df %>% unnest_tokens(trigram, text, token = "ngrams", n = 3)

# Tokenize to characters
df %>% unnest_tokens(char, text, token = "characters")

# Custom tokenizer
df %>% unnest_tokens(word, text, token = "regex", pattern = "\\s+")

Stop Words

# Remove stop words
df %>%
  unnest_tokens(word, text) %>%
  anti_join(stop_words)

# Custom stop words
custom_stops <- tibble(word = c("custom", "words"))
df %>%
  unnest_tokens(word, text) %>%
  anti_join(stop_words) %>%
  anti_join(custom_stops)

# Different lexicons
stop_words %>% filter(lexicon == "snowball")
stop_words %>% filter(lexicon == "onix")
stop_words %>% filter(lexicon == "SMART")

Word Frequencies

# Count words
word_counts <- df %>%
  unnest_tokens(word, text) %>%
  anti_join(stop_words) %>%
  count(word, sort = TRUE)

# By document
word_counts <- df %>%
  unnest_tokens(word, text) %>%
  count(document, word, sort = TRUE)

TF-IDF

# Calculate TF-IDF
tfidf <- df %>%
  unnest_tokens(word, text) %>%
  count(document, word) %>%
  bind_tf_idf(word, document, n)

# Top TF-IDF words per document
tfidf %>%
  group_by(document) %>%
  slice_max(tf_idf, n = 10)

Sentiment Analysis

# Get sentiments
get_sentiments("afinn")   # Score -5 to 5
get_sentiments("bing")    # positive/negative
get_sentiments("nrc")     # Multiple emotions

# Join sentiments
df %>%
  unnest_tokens(word, text) %>%
  inner_join(get_sentiments("bing")) %>%
  count(sentiment)

# Sentiment by document
df %>%
  unnest_tokens(word, text) %>%
  inner_join(get_sentiments("afinn")) %>%
  group_by(document) %>%
  summarize(sentiment = sum(value))

# NRC emotions
df %>%
  unnest_tokens(word, text) %>%
  inner_join(get_sentiments("nrc")) %>%
  count(sentiment, sort = TRUE)