Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
keep original signature
  • Loading branch information
yanboliang committed Dec 11, 2015
commit bf9450a62aed142b53467283fcbc2dd7c343de58
33 changes: 7 additions & 26 deletions R/pkg/R/SQLContext.R
Original file line number Diff line number Diff line change
Expand Up @@ -206,8 +206,10 @@ setMethod("toDF", signature(x = "RDD"),
#' It goes through the entire dataset once to determine the schema.
#'
#' @param sqlContext SQLContext to use
#' @param ... Path(s) of parquet file(s) to read.
#' @param path Path of file to read. A vector of multiple paths is allowed.
#' @return DataFrame
#' @rdname read.json
#' @name read.json
#' @export
#' @examples
#'\dontrun{
Expand All @@ -216,38 +218,17 @@ setMethod("toDF", signature(x = "RDD"),
#' path <- "path/to/file.json"
#' df <- read.json(sqlContext, path)
#' }

read.json <- function(sqlContext, ...) {
read.json <- function(sqlContext, path) {
# Allow the user to have a more flexible definiton of the text file path
paths <- if (length(list(...)) > 1) {
lapply(list(...), function(x) suppressWarnings(normalizePath(x)))
} else {
as.list(suppressWarnings(normalizePath(splitString(...))))
}
# Convert a string vector of paths to a string containing comma separated paths
paths <- as.list(suppressWarnings(normalizePath(splitString(path))))
read <- callJMethod(sqlContext, "read")
sdf <- callJMethod(read, "json", paths)
dataFrame(sdf)
}

#' Create a DataFrame from a JSON file.
#'
#' Loads a JSON file (one object per line), returning the result as a DataFrame
#' It goes through the entire dataset once to determine the schema. This function
#' is deprecated, please use read.json.
#'
#' @param sqlContext SQLContext to use
#' @param path Path of file to read. A vector of multiple paths is allowed.
#' @return DataFrame
#' @rdname read.json
#' @name jsonFile
#' @export
#' @examples
#'\dontrun{
#' sc <- sparkR.init()
#' sqlContext <- sparkRSQL.init(sc)
#' path <- "path/to/file.json"
#' df <- jsonFile(sqlContext, path)
#' }

jsonFile <- function(sqlContext, path) {
.Deprecated("read.json")
read.json(sqlContext, path)
Expand Down
6 changes: 3 additions & 3 deletions R/pkg/inst/tests/testthat/test_sparkSQL.R
Original file line number Diff line number Diff line change
Expand Up @@ -371,17 +371,17 @@ test_that("Collect DataFrame with complex types", {
expect_equal(bob$height, 176.5)
})

test_that("read.json() on a local file returns a DataFrame", {
test_that("read.json()/jsonFile() on a local file returns a DataFrame", {
df <- read.json(sqlContext, jsonPath)
expect_is(df, "DataFrame")
expect_equal(count(df), 3)
# read.json works with multiple input paths
jsonPath2 <- tempfile(pattern="jsonPath2", fileext=".json")
write.df(df, jsonPath2, "json", mode="overwrite")
jsonDF1 <- read.json(sqlContext, jsonPath, jsonPath2)
jsonDF1 <- read.json(sqlContext, c(jsonPath, jsonPath2))
expect_is(jsonDF1, "DataFrame")
expect_equal(count(jsonDF1), 6)
jsonDF2 <- read.json(sqlContext, c(jsonPath, jsonPath2))
jsonDF2 <- jsonFile(sqlContext, c(jsonPath, jsonPath2))
expect_is(jsonDF2, "DataFrame")
expect_equal(count(jsonDF2), 6)

Expand Down