From 148035566cf7adc0e2e4397431b35e6230116673 Mon Sep 17 00:00:00 2001 From: JulianUmbhau Date: Mon, 8 Sep 2025 09:15:55 +0000 Subject: [PATCH 1/5] labels added to query jobs --- R/query.R | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/R/query.R b/R/query.R index c43482b..8f6098e 100644 --- a/R/query.R +++ b/R/query.R @@ -3,6 +3,7 @@ #' @param projectId The BigQuery project ID #' @param datasetId A datasetId within projectId #' @param query BigQuery SQL. You can also supply a file location of your query ending with \code{.sql} +#' @param labels Labels to tag to job in list format. lower-case only. Default empty list #' @param maxResults Max number per page of results. Set total rows with LIMIT in your query. #' @param useLegacySql Whether the query you pass is legacy SQL or not. Default TRUE #' @param useQueryCache Whether to use the query cache. Default TRUE, set to FALSE for realtime queries. @@ -33,6 +34,7 @@ bqr_query <- function(projectId = bqr_get_global_project(), datasetId = bqr_get_global_dataset(), query, + labels = list(), maxResults = 1000, useLegacySql = TRUE, useQueryCache = TRUE, @@ -50,6 +52,7 @@ bqr_query <- function(projectId = bqr_get_global_project(), body <- list( kind = "bigquery#queryRequest", query = query, + labels = labels, maxResults = maxResults, useLegacySql = useLegacySql, useQueryCache = useQueryCache, @@ -125,6 +128,7 @@ bqr_query <- function(projectId = bqr_get_global_project(), #' @param projectId projectId to be billed. #' @param datasetId datasetId of where query will execute. #' @param query The BigQuery query as a string. +#' @param labels Labels to tag to job in list format. lower-case only. Default empty list #' @param destinationTableId Id of table the results will be written to. #' @param writeDisposition Behaviour if destination table exists. See Details. #' @param useLegacySql Whether the query you pass is legacy SQL or not. Default TRUE @@ -202,6 +206,7 @@ bqr_query <- function(projectId = bqr_get_global_project(), bqr_query_asynch <- function(projectId = bqr_get_global_project(), datasetId = bqr_get_global_dataset(), query, + labels = list(), destinationTableId, useLegacySql = TRUE, writeDisposition = c("WRITE_EMPTY", @@ -239,7 +244,8 @@ bqr_query_asynch <- function(projectId = bqr_get_global_project(), query = query, useLegacySql = useLegacySql, writeDisposition = writeDisposition - ) + ), + labels = labels ) ) From 4bb39637ee0caae66758244fae02e02f862bde6f Mon Sep 17 00:00:00 2001 From: JulianUmbhau Date: Mon, 8 Sep 2025 09:19:10 +0000 Subject: [PATCH 2/5] documentation --- DESCRIPTION | 2 +- man/bigQueryR.Rd | 10 ++++++++++ man/bqr_query.Rd | 3 +++ man/bqr_query_asynch.Rd | 3 +++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 454d847..3301c2a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -28,5 +28,5 @@ Suggests: testthat, data.table, purrr -RoxygenNote: 7.1.1 +RoxygenNote: 7.3.1 VignetteBuilder: knitr diff --git a/man/bigQueryR.Rd b/man/bigQueryR.Rd index 9f2370f..cef19b9 100644 --- a/man/bigQueryR.Rd +++ b/man/bigQueryR.Rd @@ -2,6 +2,7 @@ % Please edit documentation in R/bigQueryR.R \docType{package} \name{bigQueryR} +\alias{bigQueryR-package} \alias{bigQueryR} \title{bigQueryR} \description{ @@ -10,3 +11,12 @@ Provides an interface with Google BigQuery \seealso{ \url{https://cloud.google.com/bigquery/docs/reference/v2/?hl=en} } +\author{ +\strong{Maintainer}: Mark Edmondson \email{r@sunholo.com} + +Other contributors: +\itemize{ + \item Hadley Wickham \email{hadley@rstudio.com} [contributor] +} + +} diff --git a/man/bqr_query.Rd b/man/bqr_query.Rd index 13198e3..d5ebcdf 100644 --- a/man/bqr_query.Rd +++ b/man/bqr_query.Rd @@ -8,6 +8,7 @@ bqr_query( projectId = bqr_get_global_project(), datasetId = bqr_get_global_dataset(), query, + labels = list(), maxResults = 1000, useLegacySql = TRUE, useQueryCache = TRUE, @@ -22,6 +23,8 @@ bqr_query( \item{query}{BigQuery SQL. You can also supply a file location of your query ending with \code{.sql}} +\item{labels}{Labels to tag to job in list format. lower-case only. Default empty list} + \item{maxResults}{Max number per page of results. Set total rows with LIMIT in your query.} \item{useLegacySql}{Whether the query you pass is legacy SQL or not. Default TRUE} diff --git a/man/bqr_query_asynch.Rd b/man/bqr_query_asynch.Rd index 03804bf..80149c4 100644 --- a/man/bqr_query_asynch.Rd +++ b/man/bqr_query_asynch.Rd @@ -8,6 +8,7 @@ bqr_query_asynch( projectId = bqr_get_global_project(), datasetId = bqr_get_global_dataset(), query, + labels = list(), destinationTableId, useLegacySql = TRUE, writeDisposition = c("WRITE_EMPTY", "WRITE_TRUNCATE", "WRITE_APPEND") @@ -20,6 +21,8 @@ bqr_query_asynch( \item{query}{The BigQuery query as a string.} +\item{labels}{Labels to tag to job in list format. lower-case only. Default empty list} + \item{destinationTableId}{Id of table the results will be written to.} \item{useLegacySql}{Whether the query you pass is legacy SQL or not. Default TRUE} From 79510d815c903d5923d50dc8f175fde08410cbb7 Mon Sep 17 00:00:00 2001 From: JulianUmbhau Date: Wed, 24 Sep 2025 13:42:46 +0200 Subject: [PATCH 3/5] adding labels --- R/downloadData.R | 7 +++++-- R/options.R | 3 ++- R/query.R | 6 ++++-- R/tables.R | 8 ++++++-- R/utilities.R | 23 +++++++++++++++++++++++ 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/R/downloadData.R b/R/downloadData.R index 0f764a2..c3c551c 100644 --- a/R/downloadData.R +++ b/R/downloadData.R @@ -83,7 +83,9 @@ bqr_extract_data <- function(projectId = bqr_get_global_project(), compression <- match.arg(compression) destinationFormat <- match.arg(destinationFormat) - + + labels <- check_labels(getOption("bigQueryR.labels")) + check_gcs_auth() stopifnot(inherits(projectId, "character"), @@ -126,7 +128,8 @@ bqr_extract_data <- function(projectId = bqr_get_global_project(), fieldDelimiter = fieldDelimiter, destinationFormat = destinationFormat, compression = compression - ) + ), + labels = labels ) ) diff --git a/R/options.R b/R/options.R index 9c3015d..b3ba365 100644 --- a/R/options.R +++ b/R/options.R @@ -8,7 +8,8 @@ googleAuthR.webapp.client_id = "68483650948-sufabj4nq9h1hjofp03hcjhk4af93080.apps.googleusercontent.com", googleAuthR.webapp.client_secret = "0tWYjliwXD32XhvDJHTl4NgN ", googleAuthR.scopes.selected = c("https://www.googleapis.com/auth/cloud-platform"), - googleAuthR.batch_endpoint = "https://www.googleapis.com/batch/bigquery/v2" + googleAuthR.batch_endpoint = "https://www.googleapis.com/batch/bigquery/v2", + bigQueryR.labels = NULL ) options(googleAuthR.httr_oauth_cache = "bq.oauth") diff --git a/R/query.R b/R/query.R index 8f6098e..d089ad3 100644 --- a/R/query.R +++ b/R/query.R @@ -34,7 +34,6 @@ bqr_query <- function(projectId = bqr_get_global_project(), datasetId = bqr_get_global_dataset(), query, - labels = list(), maxResults = 1000, useLegacySql = TRUE, useQueryCache = TRUE, @@ -42,6 +41,8 @@ bqr_query <- function(projectId = bqr_get_global_project(), timeoutMs = 600*1000){ check_bq_auth() + labels <- check_labels(getOption("bigQueryR.labels")) + if(endsWith(query, ".sql")){ query <- readChar(query, nchars = file.info(query)$size) } @@ -206,7 +207,6 @@ bqr_query <- function(projectId = bqr_get_global_project(), bqr_query_asynch <- function(projectId = bqr_get_global_project(), datasetId = bqr_get_global_dataset(), query, - labels = list(), destinationTableId, useLegacySql = TRUE, writeDisposition = c("WRITE_EMPTY", @@ -215,6 +215,8 @@ bqr_query_asynch <- function(projectId = bqr_get_global_project(), writeDisposition <- match.arg(writeDisposition) + labels <- check_labels(getOption("bigQueryR.labels")) + check_bq_auth() ## make job job <- diff --git a/R/tables.R b/R/tables.R index 22fb0c5..149d003 100644 --- a/R/tables.R +++ b/R/tables.R @@ -27,6 +27,8 @@ bqr_copy_table <- function(source_tableid, createDisposition <- match.arg(createDisposition) writeDisposition <- match.arg(writeDisposition) + + labels <- check_labels(getOption("bigQueryR.labels")) assert_that( is.string(source_projectid), @@ -52,7 +54,8 @@ bqr_copy_table <- function(source_tableid, tableId = destination_tableid ), writeDisposition = writeDisposition - ) + ), + labels = labels ) ) @@ -413,7 +416,6 @@ Table <- function(tableId, encryptionConfiguration = NULL, expirationTime = NULL, friendlyName = NULL, - labels = NULL, materializedView = NULL, rangePartitioning = NULL, requirePartitionFilter = NULL, @@ -431,6 +433,8 @@ Table <- function(tableId, # is.flag(requirePartitionFilter) ) + labels <- check_labels(getOption("bigQueryR.labels")) + tt <- list( tableReference = list(projectId = projectId, datasetId = datasetId, diff --git a/R/utilities.R b/R/utilities.R index 6987064..fd3a464 100644 --- a/R/utilities.R +++ b/R/utilities.R @@ -137,3 +137,26 @@ idempotency <- function(){ set.seed(Sys.time()) paste(sample(c(LETTERS, letters, 0:9), 15, TRUE),collapse="") } + +#' Labels validity check +#' +#' A check of labels type and format validity +#' +#' @return labels or NULL +#' @keywords internal +#' @noRd +check_labels <- function(labels) { + if (!is.list(labels)) { + warning(paste0("Labels need to be a dictionary list - setting labels to NULL"), immediate. = TRUE) + return(NULL) + } + + for (nm in names(labels)) { + if (!is.character(labels[[nm]]) || length(labels[[nm]]) != 1) { + warning(sprintf("Label '%s' must be a single string type - setting labels to NULL", nm), immediate. = TRUE) + return(NULL) + } + } + + return(labels) +} From 39094d6a016b1fa29cca879ee5b6da9e73792b16 Mon Sep 17 00:00:00 2001 From: JulianUmbhau Date: Wed, 24 Sep 2025 13:10:04 +0000 Subject: [PATCH 4/5] fixing labels to list() --- R/options.R | 2 +- R/utilities.R | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/R/options.R b/R/options.R index b3ba365..df7f9f9 100644 --- a/R/options.R +++ b/R/options.R @@ -9,7 +9,7 @@ googleAuthR.webapp.client_secret = "0tWYjliwXD32XhvDJHTl4NgN ", googleAuthR.scopes.selected = c("https://www.googleapis.com/auth/cloud-platform"), googleAuthR.batch_endpoint = "https://www.googleapis.com/batch/bigquery/v2", - bigQueryR.labels = NULL + bigQueryR.labels = list() ) options(googleAuthR.httr_oauth_cache = "bq.oauth") diff --git a/R/utilities.R b/R/utilities.R index fd3a464..84e2c77 100644 --- a/R/utilities.R +++ b/R/utilities.R @@ -147,13 +147,20 @@ idempotency <- function(){ #' @noRd check_labels <- function(labels) { if (!is.list(labels)) { - warning(paste0("Labels need to be a dictionary list - setting labels to NULL"), immediate. = TRUE) + warning(paste0("Labels need to be a dictionary list - setting labels to list()"), immediate. = TRUE) return(NULL) } - for (nm in names(labels)) { if (!is.character(labels[[nm]]) || length(labels[[nm]]) != 1) { - warning(sprintf("Label '%s' must be a single string type - setting labels to NULL", nm), immediate. = TRUE) + warning(sprintf("Label '%s' must be a single string type - setting labels to list()", nm), immediate. = TRUE) + return(NULL) + } + if (nm != tolower(nm)) { + warning(sprintf("Label key '%s' must be lowercase - setting labels to list()", nm), immediate. = TRUE) + return(NULL) + } + if (labels[[nm]] != tolower(labels[[nm]])) { + warning(sprintf("Label value '%s' must be lowercase - setting labels to list()", labels[[nm]]), immediate. = TRUE) return(NULL) } } From 6f524039d0d269cae1207807e9034779c2b1e584 Mon Sep 17 00:00:00 2001 From: JulianUmbhau Date: Fri, 26 Sep 2025 11:17:22 +0000 Subject: [PATCH 5/5] cleanuip --- R/tables.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/R/tables.R b/R/tables.R index 149d003..db85af7 100644 --- a/R/tables.R +++ b/R/tables.R @@ -416,6 +416,7 @@ Table <- function(tableId, encryptionConfiguration = NULL, expirationTime = NULL, friendlyName = NULL, + labels = NULL, materializedView = NULL, rangePartitioning = NULL, requirePartitionFilter = NULL, @@ -433,7 +434,7 @@ Table <- function(tableId, # is.flag(requirePartitionFilter) ) - labels <- check_labels(getOption("bigQueryR.labels")) + labels <- if (!is.null(labels)) check_labels(labels) else check_labels(getOption("bigQueryR.labels")) tt <- list( tableReference = list(projectId = projectId, @@ -464,4 +465,4 @@ is.table <- function(x){ as.table <- function(x){ structure(x, class = "gar_Table") -} \ No newline at end of file +}