tokenizers

tokenizers

Fast, consistent tokenization of natural language text.

Word Tokenization

library(tokenizers)

# Tokenize into words
tokenize_words("This is a test sentence.")

# Multiple texts
texts <- c("First sentence.", "Second sentence.")
tokenize_words(texts)

Options

# Lowercase
tokenize_words(text, lowercase = TRUE)

# Keep punctuation
tokenize_words(text, strip_punct = FALSE)

# Keep numbers
tokenize_words(text, strip_numeric = FALSE)

# Stopwords
tokenize_words(text, stopwords = stopwords::stopwords("en"))

Sentence Tokenization

# Tokenize into sentences
tokenize_sentences("First sentence. Second sentence!")

# With abbreviations
tokenize_sentences(text, strip_punct = FALSE)

Character Tokenization

# Single characters
tokenize_characters("hello")

# Character shingles
tokenize_character_shingles("hello", n = 3)

N-grams

# Word n-grams
tokenize_ngrams("This is a test", n = 2)

# Range of n-grams
tokenize_ngrams("This is a test", n = 2, n_min = 1)

# Skip-grams
tokenize_skip_ngrams("This is a test", n = 2, k = 1)

Paragraph Tokenization

# Split by paragraphs
text <- "First paragraph.\n\nSecond paragraph."
tokenize_paragraphs(text)

Line Tokenization

# Split by lines
tokenize_lines("Line 1\nLine 2\nLine 3")

Regex Tokenization

# Custom pattern
tokenize_regex(text, pattern = "\\s+")

Word Stems

# Tokenize and stem
tokenize_word_stems("running cats jumping")

# With language
tokenize_word_stems(text, language = "english")

PTB Tokenization

# Penn Treebank style
tokenize_ptb("It's a test.")

Tweet Tokenization

# Twitter-aware tokenization
tokenize_tweets("Hello @user! Check out #rstats http://example.com")

Count Tokens

# Count words
count_words("This is a test sentence.")

# Count sentences
count_sentences("First. Second. Third.")

# Count characters
count_characters("hello")