Merge remote-tracking branch 'remotes/origin/master' into rebase-avro…

…-datetime

# Conflicts:
#	sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
#	sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala

@@ -937,44 +937,6 @@ object DateTimeUtils {
     new CalendarInterval(months, days, 0)
   }
 
-  /**
-   * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
-   * parsing/formatting datetime values. The pattern string is incompatible with the one defined
-   * by SimpleDateFormat in Spark 2.4 and earlier. This function converts all incompatible pattern
-   * for the new parser in Spark 3.0. See more details in SPARK-31030.
-   * @param pattern The input pattern.
-   * @return The pattern for new parser
-   */
-  def convertIncompatiblePattern(pattern: String): String = {
-    val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
-      case (patternPart, index) =>
-        // Text can be quoted using single quotes, we only check the non-quote parts.
-        index % 2 == 0 && patternPart.contains("G")
-    }
-    pattern.split("'").zipWithIndex.map {
-      case (patternPart, index) =>
-        if (index % 2 == 0) {
-          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
-          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
-          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
-          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
-          // exception and ask users to change the pattern strings or turn on the legacy mode;
-          // otherwise, return NULL as what Spark 2.4 does.
-          val res = patternPart.replace("u", "e")
-          // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
-          // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
-          // We only do this substitution when there is no era designator found in the pattern.
-          if (!eraDesignatorContained) {
-            res.replace("y", "u")
-          } else {
-            res
-          }
-        } else {
-          patternPart
-        }
-    }.mkString("'")
-  }
-
   /**
    * Converts the given microseconds to a local date-time in UTC time zone in Proleptic Gregorian
    * calendar, interprets the result as a local date-time in Julian calendar in UTC time zone.

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala

@@ -671,22 +671,6 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     }
   }
 
-  test("check incompatible pattern") {
-    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
-    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
-    assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
-      === "uuuu-MM'y contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
-    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
-      === "uuuu-MM'u contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
-      === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
-      === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
-  }
-
   test("rebase julian to/from gregorian micros") {
     outstandingTimezones.foreach { timeZone =>
       withDefaultTimeZone(timeZone) {