Access workspace bucket data from R

This notebook is the R version of the Python workflow that uses maap.aws.workspace_bucket_credentials() to retrieve temporary AWS credentials for MAAP workspace and organization S3 buckets.

The access level is controlled by MAAP and is returned in authorized_s3_paths. This notebook retrieves those temporary credentials and shows two R access patterns:

  1. Update a named AWS profile in ~/.aws/credentials.

  2. Inject the same temporary credentials directly into the current GDAL/VSI environment for /vsis3/ reads.

The profile-based method is useful because many GDAL-backed R packages can read from the standard AWS credentials file. The GDAL/VSI environment method is useful for session-only access because it avoids depending on a stored profile.

1. Retrieve temporary credentials

[1]:

library(httr2)

profile_name <- "geotrees"
aws_region <- "us-west-2"

# Use the MAAP API host, not the MAAP Hub website.
maap_api_host <- "https://api.maap-project.org"

The MAAP Hub website, such as https://hub.maap-project.org, is the JupyterHub/ADE user interface. The credential request should go to the MAAP API service.

[2]:

trim_slash <- function(x) {
  gsub("^/+|/+$", "", x)
}

join_url <- function(base, path) {
  paste0(gsub("/+$", "", base), "/", trim_slash(path))
}

get_maap_config <- function(maap_api_host = "https://api.maap-project.org") {
  config_url <- join_url(maap_api_host, "api/environment/config")

  resp <- request(config_url) |>
    req_headers(Accept = "application/json") |>
    req_perform()

  resp_body_json(resp, simplifyVector = FALSE)
}

build_maap_endpoint <- function(config, maap_api_host, endpoint_key) {
  api_root <- config$service$maap_api_root

  # If the config gives only a path like "/api", attach it to the API host.
  # If it gives a full URL, use it as-is.
  if (!grepl("^https?://", api_root)) {
    api_root <- join_url(maap_api_host, api_root)
  }

  endpoint_path <- config$maap_endpoint[[endpoint_key]]

  if (is.null(endpoint_path)) {
    stop(paste("Endpoint key not found in MAAP config:", endpoint_key))
  }

  join_url(api_root, endpoint_path)
}

get_workspace_bucket_credentials <- function(
    maap_api_host = "https://api.maap-project.org"
) {
  config <- get_maap_config(maap_api_host)

  token <- config$service$maap_token

  if (is.null(token) || token == "") {
    stop("Could not find maap_token from MAAP environment config.")
  }

  endpoint <- build_maap_endpoint(
    config = config,
    maap_api_host = maap_api_host,
    endpoint_key = "workspace_bucket_credentials"
  )

  headers <- list(
    Accept = "application/json",
    token = token
  )

  # Include proxy ticket if the environment provides one.
  maap_pgt <- Sys.getenv("MAAP_PGT")
  if (maap_pgt != "") {
    headers[["proxy-ticket"]] <- maap_pgt
  }

  resp <- request(endpoint) |>
    req_headers(!!!headers) |>
    req_perform()

  resp_body_json(resp, simplifyVector = FALSE)
}

[ ]:

resp <- get_workspace_bucket_credentials(maap_api_host)

str(resp, max.level = 2)

The response contains:

  • credentials — temporary AWS credentials:

    • aws_access_key_id

    • aws_secret_access_key

    • aws_session_token

    • expires_at

  • authorized_s3_paths — S3 paths that the credentials can access:

    • bucket

    • prefix

    • uri

    • type

    • access

2. Create or update an AWS profile from the credentials

The Python workflow creates a boto3.Session from the returned credentials.

In R, one practical equivalent for GDAL-backed packages is to write the temporary credentials to the standard AWS credentials file:

~/.aws/credentials

The credentials are written under the profile name geotrees.

If the file already exists, this notebook does not overwrite the whole file. It updates only the [geotrees] profile block. Other AWS profiles already present in ~/.aws/credentials are preserved.

If [geotrees] already exists, its access key, secret key, and session token are replaced with the latest temporary credentials. This is important because MAAP returns temporary credentials that expire and need to be refreshed.

The directory and file permissions are also set:

  • ~/.aws/ directory: 0700

  • ~/.aws/credentials file: 0600

[4]:

get_value <- function(x, names) {
  for (name in names) {
    if (!is.null(x[[name]])) {
      return(x[[name]])
    }
  }

  NULL
}

replace_or_append_profile <- function(existing_lines, profile_name, profile_lines) {
  profile_header <- paste0("[", profile_name, "]")
  header_pattern <- "^\\s*\\[[^]]+\\]\\s*$"

  if (length(existing_lines) == 0) {
    return(profile_lines)
  }

  header_locations <- grep(header_pattern, existing_lines)
  target_start <- which(trimws(existing_lines) == profile_header)

  if (length(target_start) == 0) {
    # Profile does not exist yet. Preserve the existing file and append the new profile.
    return(c(existing_lines, "", profile_lines))
  }

  target_start <- target_start[1]
  later_headers <- header_locations[header_locations > target_start]
  target_end <- if (length(later_headers) > 0) later_headers[1] - 1 else length(existing_lines)

  before <- if (target_start > 1) existing_lines[1:(target_start - 1)] else character()
  after <- if (target_end < length(existing_lines)) existing_lines[(target_end + 1):length(existing_lines)] else character()

  c(before, profile_lines, after)
}

extract_aws_credentials <- function(resp) {
  # MAAP may return credentials either at the top level or nested under "credentials".
  creds <- resp$credentials
  if (is.null(creds)) {
    creds <- resp
  }

  access_key <- get_value(
    creds,
    c("aws_access_key_id", "accessKeyId", "AccessKeyId")
  )

  secret_key <- get_value(
    creds,
    c("aws_secret_access_key", "secretAccessKey", "SecretAccessKey")
  )

  session_token <- get_value(
    creds,
    c("aws_session_token", "sessionToken", "SessionToken")
  )

  if (is.null(access_key) || is.null(secret_key) || is.null(session_token)) {
    print(resp)
    stop("Could not find AWS credential fields in MAAP response.")
  }

  list(
    access_key = access_key,
    secret_key = secret_key,
    session_token = session_token,
    expires_at = get_value(creds, c("expires_at", "Expiration", "expiration"))
  )
}

write_aws_credentials_profile <- function(resp, profile_name = "geotrees") {
  creds <- extract_aws_credentials(resp)

  aws_dir <- path.expand("~/.aws")
  credentials_file <- file.path(aws_dir, "credentials")

  if (!dir.exists(aws_dir)) {
    dir.create(aws_dir, recursive = TRUE, mode = "0700")
  }

  profile_lines <- c(
    paste0("[", profile_name, "]"),
    paste0("aws_access_key_id = ", creds$access_key),
    paste0("aws_secret_access_key = ", creds$secret_key),
    paste0("aws_session_token = ", creds$session_token)
  )

  existing_lines <- if (file.exists(credentials_file)) {
    readLines(credentials_file, warn = FALSE)
  } else {
    character()
  }

  updated_lines <- replace_or_append_profile(
    existing_lines = existing_lines,
    profile_name = profile_name,
    profile_lines = profile_lines
  )

  writeLines(updated_lines, credentials_file)
  Sys.chmod(aws_dir, mode = "0700")
  Sys.chmod(credentials_file, mode = "0600")

  invisible(credentials_file)
}

[ ]:

credentials_file <- write_aws_credentials_profile(
  resp = resp,
  profile_name = profile_name
)

Sys.setenv(
  AWS_PROFILE = profile_name,
  AWS_DEFAULT_REGION = aws_region,
  AWS_SDK_LOAD_CONFIG = "1",
  AWS_NO_SIGN_REQUEST = "NO"
)

cat("AWS credentials profile created or updated successfully.\n")
cat("Profile:", profile_name, "\n")
cat("Credentials file:", credentials_file, "\n")

creds <- extract_aws_credentials(resp)
if (!is.null(creds$expires_at)) {
  cat("Expires at:", creds$expires_at, "\n")
}

Confirm that the profile is available in the R session.

Do not print the full credentials file because it contains temporary secrets. The check below only verifies that the selected profile header exists.

[ ]:

Sys.getenv("AWS_PROFILE")
Sys.getenv("AWS_DEFAULT_REGION")
file.exists(path.expand("~/.aws/credentials"))
any(readLines(path.expand("~/.aws/credentials"), warn = FALSE) == paste0("[", profile_name, "]"))

3. Optional: inject temporary credentials into the current GDAL/VSI session

Another possible approach is to pass the temporary credentials directly into the current R session before opening /vsis3/ paths.

This may work well for GDAL-backed packages because they can read S3 authentication information from environment variables or GDAL configuration values during the current session.

This method is useful when you do not want to depend on a profile file. It should still be tested in the target MAAP Hub image with the packages you plan to use.

[ ]:

set_gdal_aws_env <- function(resp, aws_region = "us-west-2") {
  creds <- extract_aws_credentials(resp)

  Sys.setenv(
    AWS_ACCESS_KEY_ID = creds$access_key,
    AWS_SECRET_ACCESS_KEY = creds$secret_key,
    AWS_SESSION_TOKEN = creds$session_token,
    AWS_DEFAULT_REGION = aws_region,
    AWS_REGION = aws_region,
    AWS_NO_SIGN_REQUEST = "NO"
  )

  invisible(TRUE)
}

set_gdal_aws_env(resp, aws_region = aws_region)

cat("Temporary AWS credentials were added to the current R session environment.\n")
cat("This can be tested with GDAL/VSI reads from terra, lasR, stars, and sf.\n")

4. Working with your workspace bucket

The Python example uses the first entry in authorized_s3_paths as the workspace bucket. We do the same here.

[ ]:

workspace <- resp$authorized_s3_paths[[1]]

workspace_bucket <- workspace$bucket
workspace_prefix <- workspace$prefix
workspace_uri <- workspace$uri
workspace_access <- workspace$access

cat("Workspace bucket:", workspace_bucket, "\n")
cat("Workspace prefix:", workspace_prefix, "\n")
cat("Workspace URI:", workspace_uri, "\n")
cat("Access:", workspace_access, "\n")

For GDAL-backed R packages such as terra and lasR, convert the S3 URI into GDAL’s /vsis3/ format. This same /vsis3/ path may also work with stars and sf, but those package-specific reads should be confirmed in the MAAP Hub image being documented.

[ ]:

workspace_vsis3 <- sub("^s3://", "/vsis3/", workspace_uri)

workspace_vsis3

Read a raster from the workspace bucket

Use a real object that exists in your authorized workspace path.

[ ]:

library(terra)

# Replace example.tif with a real file inside your authorized workspace path.
s3_path <- file.path(workspace_uri, "example.tif")
vsis3_path <- sub("^s3://", "/vsis3/", s3_path)

# Uncomment after replacing example.tif with a real object.
# r <- rast(vsis3_path)
# r

Upload or write a file to the workspace bucket

Writing only works when the path has access = "read_write".

The example below creates a small raster, writes it to the workspace bucket, and reads it back. This is the R/GDAL equivalent of using s3.upload_file() in the Python example.

[ ]:

library(terra)

if (workspace_access == "read_write") {
  r <- rast(
    nrows = 10,
    ncols = 10,
    xmin = 0,
    xmax = 10,
    ymin = 0,
    ymax = 10
  )

  values(r) <- 1:ncell(r)

  output_path <- file.path(workspace_vsis3, "r_credential_test.tif")

  writeRaster(r, output_path, overwrite = TRUE)

  r_check <- rast(output_path)
  r_check
} else {
  cat("Workspace path is not writable. Access level:", workspace_access, "\n")
}

5. Working with organization shared buckets

Additional organization-granted buckets appear as extra entries in authorized_s3_paths. Each entry tells you whether it is read_write or read_only.

[ ]:

for (path in resp$authorized_s3_paths) {
  cat(path$uri, "(", path$access, ")", "\n")
}

[ ]:

org_paths <- Filter(
  function(path) {
    !is.null(path$type) && path$type == "org"
  },
  resp$authorized_s3_paths
)

length(org_paths)

Select the first organization path, if one is available.

[ ]:

if (length(org_paths) > 0) {
  shared <- org_paths[[1]]

  shared_bucket <- shared$bucket
  shared_prefix <- shared$prefix
  shared_uri <- shared$uri
  shared_access <- shared$access
  shared_vsis3 <- sub("^s3://", "/vsis3/", shared_uri)

  cat("Shared bucket:", shared_bucket, "\n")
  cat("Shared prefix:", shared_prefix, "\n")
  cat("Shared URI:", shared_uri, "\n")
  cat("Shared /vsis3/ path:", shared_vsis3, "\n")
  cat("Access:", shared_access, "\n")
} else {
  cat("No organization shared buckets were returned.\n")
}

To read from a shared bucket, use a real object path inside the shared prefix. To write to a shared bucket, first check that access is read_write.

[14]:

if (length(org_paths) > 0) {
  shared_file <- file.path(shared_vsis3, "shared_dataset.tif")

  # Uncomment after replacing shared_dataset.tif with a real object.
  # shared_raster <- rast(shared_file)
  # shared_raster

  if (shared_access == "read_write") {
    # Example write path. Uncomment when ready to write.
    # shared_output <- file.path(shared_vsis3, "my_output.tif")
    # writeRaster(r, shared_output, overwrite = TRUE)
  }
}