Support other formats of recordDelimiter

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala

@@ -98,13 +98,14 @@ private[sql] class JSONOptions(
    * - 'x' + sequence of bytes in hexadecimal format. For example: "x0a 0d".
    *   Hex pairs can be separated by any chars different from 0-9,A-F,a-f
    * - '\' - reserved for a sequence of control chars like "\r\n"
-   * - '/' - reserved for a sequence of visible chars like "/==="
+   *         and unicode escape like "\u000D\u000A"
    */
   val recordDelimiter: Option[Array[Byte]] = parameters.get("recordDelimiter").collect {
     case hexs if hexs.startsWith("x") =>
       hexs.replaceAll("[^0-9A-Fa-f]", "").sliding(2, 2).toArray
         .map(Integer.parseInt(_, 16).toByte)
-    case d => throw new NotImplementedError(d)
+    case delim => delim.getBytes(charset.getOrElse(
+      throw new IllegalArgumentException("Please, set the charset option for the delimiter")))
   }
 
   /** Sets config options on a Jackson [[JsonFactory]]. */

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala

@@ -2063,4 +2063,31 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
       )
     }
   }
+
+  def readWrittenJson(delimiter: (String, Int)): Unit = {
+    val (recordDelimiter, index) = delimiter
+    test(s"read written json in UTF-16BE with delimiter $index") {
+      val charset = "UTF-16BE"
+      case class Rec(f1: String, f2: Int)
+      withTempPath { path =>
+        val ds = spark.createDataset(Seq(
+          ("a", 1), ("b", 2), ("c", 3))
+        ).repartition(1)
+        ds.write
+          .option("charset", charset)
+          .format("json").mode("overwrite")
+          .save(path.getCanonicalPath)
+        val savedDf = spark
+          .read
+          .schema(ds.schema)
+          .option("charset", charset)
+          .option("recordDelimiter", recordDelimiter)
+          .json(path.getCanonicalPath)
+
+        checkAnswer(savedDf.toDF(), ds.toDF())
+      }
+    }
+  }
+
+  List("x00 0a", "\n", "\u000a").zipWithIndex.foreach(readWrittenJson(_))
 }