Web APIs, Scraping & JSON

41.1 Calling APIs with httr2

Older: httr, curl, crul. Store secrets in .Renviron (usethis::edit_r_environ()), read with Sys.getenv().

library(httr2)

resp <- request("https://api.github.com") |>
  req_url_path_append("repos", "tidyverse", "dplyr") |>
  req_headers(Accept = "application/vnd.github+json") |>
  req_user_agent("my-r-course") |>
  req_retry(max_tries = 3) |>
  req_perform()

resp_status(resp)
repo <- resp_body_json(resp)
repo$stargazers_count

# POST with JSON body & auth
request("https://httpbin.org/post") |>
  req_body_json(list(name = "Ann", score = 9)) |>
  req_auth_bearer_token(Sys.getenv("API_TOKEN")) |>
  req_perform() |> resp_body_json()

# Pagination, rate limits, OAuth: req_perform_iterative(), req_throttle(), req_oauth_auth_code()

41.2 JSON

Nested JSON → tidyr::unnest_wider()/unnest_longer()/hoist(); also tidyjson, RcppSimdJson (fast).

library(jsonlite)
fromJSON('{"name": "Ann", "langs": ["R", "SQL"]}')          # list
fromJSON('[{"a":1,"b":2},{"a":3,"b":4}]')                   # data frame
toJSON(head(mtcars, 2), pretty = TRUE, auto_unbox = TRUE)
fromJSON(url, flatten = TRUE)
stream_in(file("big.ndjson"))                               # newline-delimited JSON

41.3 Web scraping with rvest

Be polite: respect robots.txt and terms of service, rate-limit (polite package), cache results.

library(rvest)
page <- read_html("https://en.wikipedia.org/wiki/R_(programming_language)")
page |> html_element("h1") |> html_text2()
page |> html_elements("table.wikitable") |> html_table()     # list of data frames
page |> html_elements("a") |> html_attr("href") |> head()
page |> html_elements(".mw-parser-output p") |> html_text2() |> head(3)

# Dynamic (JavaScript) pages
sess <- read_html_live("https://example.com")   # rvest + chromote headless browser

41.4 Building APIs with plumber

Alternatives: ambiorix, RestRserve, vetiver (deploys tidymodels as plumber APIs).

# plumber.R
#* Echo back the input
#* @param msg The message
#* @get /echo
function(msg = "") list(msg = paste("You said:", msg))

#* Predict mpg from weight
#* @param wt:numeric
#* @post /predict
function(wt) {
  model <- readRDS("model.rds")
  list(mpg = predict(model, data.frame(wt = as.numeric(wt))))
}

#* Plot
#* @serializer png
#* @get /plot
function() plot(mtcars$wt, mtcars$mpg)

plumber::pr("plumber.R") |> plumber::pr_run(port = 8000)   # Swagger docs at /__docs__/

41.5 Email, Slack, cloud

blastula/emayili (email), slackr, googledrive/googlesheets4, aws.s3/paws (AWS), AzureStor, bigrquery, gargle (Google auth).