arrow

arrow

Apache Arrow interface.

Read/Write Parquet

library(arrow)

# Read
df <- read_parquet("file.parquet")
df <- read_parquet("file.parquet", col_select = c("a", "b"))

# Write
write_parquet(df, "output.parquet")
write_parquet(df, "output.parquet", compression = "snappy")
write_parquet(df, "output.parquet", compression = "gzip")

Read/Write Feather

# Read
df <- read_feather("file.feather")

# Write
write_feather(df, "output.feather")
write_feather(df, "output.feather", compression = "lz4")

Datasets (Multiple Files)

# Open dataset
ds <- open_dataset("data/")
ds <- open_dataset("data/", format = "parquet")
ds <- open_dataset("data/", format = "csv")

# Query with dplyr
result <- ds %>%
  filter(year == 2024) %>%
  select(a, b, c) %>%
  collect()

# Partitioned data
ds <- open_dataset("data/", partitioning = c("year", "month"))

# Write partitioned
write_dataset(
  df,
  "output/",
  format = "parquet",
  partitioning = c("year", "month")
)

Arrow Tables

# Create Arrow table
tbl <- arrow_table(df)
tbl <- as_arrow_table(df)

# Convert back
df <- as.data.frame(tbl)

# Schema
schema(tbl)
tbl$schema

# Select columns
tbl$a
tbl[c("a", "b")]

Data Types

# Schema specification
schema(
  id = int32(),
  name = utf8(),
  value = float64(),
  date = date32(),
  timestamp = timestamp("us", timezone = "UTC"),
  flag = boolean()
)

# Cast types
tbl$cast(schema(...))

Streaming

# Read in batches
reader <- ParquetFileReader$create("file.parquet")
batch <- reader$ReadRowGroup(0)

# Write in batches
writer <- ParquetFileWriter$create(
  "output.parquet",
  schema = schema(...)
)
writer$WriteTable(tbl)
writer$Close()

CSV with Arrow

# Fast CSV reading
df <- read_csv_arrow("file.csv")
df <- read_csv_arrow("file.csv", col_types = schema(
  id = int32(),
  name = utf8()
))

# Write CSV
write_csv_arrow(df, "output.csv")

Memory Mapping

# Memory-mapped file
tbl <- read_parquet("file.parquet", as_data_frame = FALSE)

# Query without loading all data
result <- tbl %>%
  filter(x > 100) %>%
  collect()