init

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala

@@ -748,6 +748,10 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
       values.map(value => s"$name = $value").mkString("(", " or ", ")")
     }
 
+    def convertNotInToAnd(name: String, values: Seq[String]): String = {
+      values.map(value => s"$name != $value").mkString("(", " and ", ")")
+    }
+
     val useAdvanced = SQLConf.get.advancedPartitionPredicatePushdownEnabled
     val inSetThreshold = SQLConf.get.metastorePartitionPruningInSetThreshold
 
@@ -767,6 +771,10 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
           if useAdvanced =>
         Some(convertInToOr(name, values))
 
+      case Not(In(ExtractAttribute(SupportedAttribute(name)), ExtractableLiterals(values)))
+        if useAdvanced =>
+        Some(convertNotInToAnd(name, values))
+
       case InSet(child, values) if useAdvanced && values.size > inSetThreshold =>
         val dataType = child.dataType
         // Skip null here is safe, more details could see at ExtractableLiterals.
@@ -779,10 +787,18 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
           if useAdvanced && child.dataType == DateType =>
         Some(convertInToOr(name, values))
 
+      case Not(InSet(child @ ExtractAttribute(SupportedAttribute(name)),
+        ExtractableDateValues(values))) if useAdvanced && child.dataType == DateType =>
+        Some(convertNotInToAnd(name, values))
+
       case InSet(ExtractAttribute(SupportedAttribute(name)), ExtractableValues(values))
           if useAdvanced =>
         Some(convertInToOr(name, values))
 
+      case Not(InSet(ExtractAttribute(SupportedAttribute(name)), ExtractableValues(values)))
+        if useAdvanced =>
+        Some(convertNotInToAnd(name, values))
+
       case op @ SpecialBinaryComparison(
           ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
         Some(s"$name ${op.symbol} $value")

sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala

@@ -108,6 +108,28 @@ class FiltersSuite extends SparkFunSuite with Logging with PlanTest {
     (a("datecol", DateType) =!= Literal(Date.valueOf("2019-01-01"))) :: Nil,
     "datecol != 2019-01-01")
 
+  filterTest("not-in int filter",
+    (Not(In(a("intcol", IntegerType), Seq(Literal(1), Literal(2))))) :: Nil,
+    "(intcol != 1 and intcol != 2)")
+
+  filterTest("not-in string filter",
+    (Not(In(a("strcol", StringType), Seq(Literal("a"), Literal("b"))))) :: Nil,
+    """(strcol != "a" and strcol != "b")""")
+
+  filterTest("not-inset, int filter",
+    (Not(InSet(a("intcol", IntegerType), Set(1, 2)))) :: Nil,
+    "(intcol != 1 and intcol != 2)")
+
+  filterTest("not-inset, string filter",
+    (Not(InSet(a("strcol", StringType), Set(Literal("a").eval(), Literal("b").eval())))) :: Nil,
+    """(strcol != "a" and strcol != "b")""")
+
+  filterTest("not-inset, date filter",
+    (Not(InSet(a("datecol", DateType),
+      Set(Literal(Date.valueOf("2020-01-01")).eval(),
+        Literal(Date.valueOf("2020-01-02")).eval())))) :: Nil,
+    """(datecol != "2020-01-01" and datecol != "2020-01-02")""")
+
   // Applying the predicate `x IN (NULL)` should return an empty set, but since this optimization
   // will be applied by Catalyst, this filter converter does not need to account for this.
   filterTest("SPARK-24879 IN predicates with only NULLs will not cause a NPE",

sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala

@@ -418,6 +418,52 @@ class HivePartitionFilteringSuite(version: String)
       dateStrValue)
   }
 
+  test("getPartitionsByFilter: not in chunk") {
+    testMetastorePartitionFiltering(
+      Not(In(attr("chunk"), Seq(Literal("aa"), Literal("ab")))),
+      dsValue,
+      hValue,
+      Seq("ba", "bb"),
+      dateValue,
+      dateStrValue
+    )
+  }
+
+  test("getPartitionsByFilter: not in ds") {
+    testMetastorePartitionFiltering(
+      Not(In(attr("ds"), Seq(Literal(20170102)))),
+      Seq(20170101, 20170103),
+      hValue,
+      chunkValue,
+      dateValue,
+      dateStrValue
+    )
+  }
+
+  test("getPartitionsByFilter: not inset date") {
+    testMetastorePartitionFiltering(
+      Not(InSet(attr("d"),
+        Set(Literal(Date.valueOf("2019-01-01")).eval(),
+          Literal(Date.valueOf("2019-01-02")).eval()))),
+      dsValue,
+      hValue,
+      chunkValue,
+      Seq("2019-01-03") ++ defaultPartition,
+      dateStrValue
+    )
+  }
+
+  test("getPartitionsByFilter: not inset h") {
+    testMetastorePartitionFiltering(
+      Not(InSet(attr("h"), Set(1, 2))),
+      dsValue,
+      Seq(0, 3, 4),
+      chunkValue,
+      dateValue,
+      dateStrValue
+    )
+  }
+
   test("getPartitionsByFilter: cast(datestr as date)= 2020-01-01") {
     testMetastorePartitionFiltering(
       attr("datestr").cast(DateType) === Date.valueOf("2020-01-01"),