Rewrite get_robotstxt_http_get

R/get_robotstxt_http_get.R

@@ -1,53 +1,35 @@
-#' Storage for http request response objects
+#' Storage for HTTP request response objects
 #'
 #' @rdname get_robotstxt_http_get
-#'
 #' @export
-rt_last_http         <- new.env()
+rt_last_http <- new.env()
 rt_last_http$request <- list()
 
 #' Execute HTTP request for get_robotstxt()
 #'
 #' @param ssl_verifypeer either 1 (default) or 0, if 0 it disables SSL peer verification, which
 #'   might help with robots.txt file retrieval
-#' @param domain the domain to get robots.txt. file for
-#' @param user_agent the user agent to use for HTTP request header
-#'
+#' @param domain the domain to get robots.txt file for.
+#' @param user_agent the user agent to use for HTTP request header. Defaults to current version of R.
+#'   If `NULL` is passed, httr will use software versions for the header, such as
+#'   `libcurl/8.7.1 r-curl/5.2.3 httr/1.4.7`
 #' @export
-#'
-get_robotstxt_http_get <-
-  function(
-    domain,
-    user_agent     = utils::sessionInfo()$R.version$version.string,
-    ssl_verifypeer = 1
-  ){
-    # generate url
-    url <- fix_url(paste0(domain, "/robots.txt"))
+get_robotstxt_http_get <- function(domain, user_agent = utils::sessionInfo()$R.version$version.string, ssl_verifypeer = 1) {
+  url <- fix_url(paste0(domain, "/robots.txt"))
 
-    if ( !is.null(user_agent) ) {
-      # with user agent
-      request <-
-        httr::GET(
-          url    = url,
-          config =
-            httr::add_headers(
-              "user-agent" = user_agent
-            ),
-          httr::config(ssl_verifypeer = ssl_verifypeer)
-        )
-    }else{
-      # without user agent
-      request <-
-        httr::GET(
-          url = url,
-          httr::config(ssl_verifypeer = ssl_verifypeer)
-        )
-    }
+  headers <- if (!is.null(user_agent)) {
+    httr::add_headers("user-agent" = user_agent)
+  } else {
+    NULL
+  }
 
+  request <- httr::GET(
+    url = url,
+    config = httr::config(ssl_verifypeer = ssl_verifypeer),
+    headers
+  )
 
-    # store in storage
-    rt_last_http$request <- request
+  rt_last_http$request <- request
 
-    # return
-    request
-  }
+  request
+}