From f61d7c26a58a3be9d84e5469a503a4be57376340 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 5 Sep 2019 15:28:22 +0800 Subject: [PATCH 01/13] add spark's type --- .../SparkGetTypeInfoOperation.scala | 104 +++++ .../sql/hive/thriftserver/cli/Type.scala | 377 ++++++++++++++++++ .../server/SparkSQLOperationManager.scala | 11 + .../cli/operation/GetTypeInfoOperation.java | 2 +- 4 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala create mode 100644 sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala new file mode 100644 index 000000000000..215475f2524f --- /dev/null +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.thriftserver + +import java.util.UUID + +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType +import org.apache.hive.service.cli.{HiveSQLException, OperationState} +import org.apache.hive.service.cli.operation.GetTypeInfoOperation +import org.apache.hive.service.cli.session.HiveSession + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SQLContext +import org.apache.spark.sql.hive.thriftserver.cli.Type +import org.apache.spark.util.{Utils => SparkUtils} + +/** + * Spark's own GetTypeInfoOperation + * + * @param sqlContext SQLContext to use + * @param parentSession a HiveSession from SessionManager + */ +private[hive] class SparkGetTypeInfoOperation( + sqlContext: SQLContext, + parentSession: HiveSession) + extends GetTypeInfoOperation(parentSession) with Logging { + + private var statementId: String = _ + + override def close(): Unit = { + super.close() + HiveThriftServer2.listener.onOperationClosed(statementId) + } + + override def runInternal(): Unit = { + statementId = UUID.randomUUID().toString + val logMsg = "Listing type info" + logInfo(s"$logMsg with $statementId") + setState(OperationState.RUNNING) + // Always use the latest class loader provided by executionHive's state. + val executionHiveClassLoader = sqlContext.sharedState.jarClassLoader + Thread.currentThread().setContextClassLoader(executionHiveClassLoader) + + if (isAuthV2Enabled) { + authorizeMetaGets(HiveOperationType.GET_TYPEINFO, null) + } + + HiveThriftServer2.listener.onStatementStart( + statementId, + parentSession.getSessionHandle.getSessionId.toString, + logMsg, + statementId, + parentSession.getUsername) + + try { + Type.types.foreach(typeInfo => { + val rowData = Array[AnyRef]( + typeInfo.getName, // TYPE_NAME + typeInfo.toJavaSQLType.asInstanceOf[AnyRef], // DATA_TYPE + typeInfo.getMaxPrecision.getOrElse(null).asInstanceOf[AnyRef], // PRECISION + typeInfo.getLiteralPrefix, // LITERAL_PREFIX + typeInfo.getLiteralSuffix, // LITERAL_SUFFIX + typeInfo.getCreateParams, // CREATE_PARAMS + typeInfo.getNullable.asInstanceOf[AnyRef], // NULLABLE + typeInfo.isCaseSensitive.asInstanceOf[AnyRef], // CASE_SENSITIVE + typeInfo.getSearchable.asInstanceOf[AnyRef], // SEARCHABLE + typeInfo.isUnsignedAttribute.asInstanceOf[AnyRef], // UNSIGNED_ATTRIBUTE + typeInfo.isFixedPrecScale.asInstanceOf[AnyRef], // FIXED_PREC_SCALE + typeInfo.isAutoIncrement.asInstanceOf[AnyRef], // AUTO_INCREMENT + typeInfo.getLocalizedName, // LOCAL_TYPE_NAME + typeInfo.getMinimumScale.asInstanceOf[AnyRef], // MINIMUM_SCALE + typeInfo.getMaximumScale.asInstanceOf[AnyRef], // MAXIMUM_SCALE + null, // SQL_DATA_TYPE, unused + null, // SQL_DATETIME_SUB, unused + typeInfo.getNumPrecRadix // NUM_PREC_RADIX + ) + rowSet.addRow(rowData) + }) + setState(OperationState.FINISHED) + } catch { + case e: HiveSQLException => + setState(OperationState.ERROR) + HiveThriftServer2.listener.onStatementError( + statementId, e.getMessage, SparkUtils.exceptionString(e)) + throw e + } + HiveThriftServer2.listener.onStatementFinish(statementId) + } +} diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala new file mode 100644 index 000000000000..a2ca981bc2b0 --- /dev/null +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.thriftserver.cli + +import java.sql.DatabaseMetaData + +import org.apache.spark.sql.types.DecimalType + +trait Type { + + def name: String + + def isComplex: Boolean + + def isPrimitiveType: Boolean = !isComplex + + def isComplexType: Boolean = isComplex + + def isQualifiedType: Boolean + + def isCollectionType: Boolean + + /** + * Radix for this type (typically either 2 or 10) + * Null is returned for data types where this is not applicable. + */ + def getNumPrecRadix: Integer = { + if (this.isNumericType) { + 10 + } else { + null + } + } + + def getMaxPrecision(): Option[Int] + + def isNumericType(): Boolean = false + + /** + * Prefix used to quote a literal of this type (may be null) + */ + def getLiteralPrefix: String = null + + /** + * Suffix used to quote a literal of this type (may be null) + * + * @return + */ + def getLiteralSuffix: String = null + + /** + * Can you use NULL for this type? + * + * @return + * DatabaseMetaData.typeNoNulls - does not allow NULL values + * DatabaseMetaData.typeNullable - allows NULL values + * DatabaseMetaData.typeNullableUnknown - nullability unknown + */ + def getNullable: Short = { + // All Hive types are nullable + DatabaseMetaData.typeNullable.toShort + } + + /** + * Is the type case sensitive? + * + * @return + */ + def isCaseSensitive: Boolean = false + + /** + * Parameters used in creating the type (may be null) + * + * @return + */ + def getCreateParams: String = null + + /** + * Can you use WHERE based on this type? + * + * @return + * DatabaseMetaData.typePredNone - No support + * DatabaseMetaData.typePredChar - Only support with WHERE .. LIKE + * DatabaseMetaData.typePredBasic - Supported except for WHERE .. LIKE + * DatabaseMetaData.typeSearchable - Supported for all WHERE .. + */ + def getSearchable: Short = { + if (isPrimitiveType) { + DatabaseMetaData.typeSearchable.toShort + } else { + DatabaseMetaData.typePredNone.toShort + } + } + + /** + * Is this type unsigned? + * + * @return + */ + def isUnsignedAttribute: Boolean = { + if (isNumericType) { + return false + } + true + } + + /** + * Can this type represent money? + * + * @return + */ + def isFixedPrecScale: Boolean = false + + /** + * Can this type be used for an auto-increment value? + * + * @return + */ + def isAutoIncrement: Boolean = false + + /** + * Localized version of type name (may be null). + * + * @return + */ + def getLocalizedName: String = null + + /** + * Minimum scale supported for this type + * + * @return + */ + def getMinimumScale: Short = 0.toShort + + /** + * Maximum scale supported for this type + * + * @return + */ + def getMaximumScale: Short = 0.toShort + + def javaSQLType: Int + + def toJavaSQLType: Int = javaSQLType + + def getName: String = name +} + +object Type { + + def types: Seq[Type] = + Seq(NULL, + STRING, + INT, + BOOLEAN, + DOUBLE, + FLOAT, + DECIMAL, + LONG, + BYTE, + SHORT, + DATE, + TIMESTAMP, + BINARY) + + case object NULL extends Type() { + override def name: String = "NULL" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.NULL + } + + case object STRING extends Type { + override def name: String = "STRING" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.VARCHAR + + override def isCaseSensitive: Boolean = true + } + + case object INT extends Type { + override def name: String = "INT" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = Some(10) + + override def javaSQLType: Int = java.sql.Types.INTEGER + + override def isNumericType(): Boolean = true + } + + case object BOOLEAN extends Type { + override def name: String = "BOOLEAN" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.BOOLEAN + } + + case object DOUBLE extends Type { + override def name: String = "DOUBLE" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = Some(15) + + override def javaSQLType: Int = java.sql.Types.DOUBLE + + override def isNumericType(): Boolean = true + } + + case object FLOAT extends Type { + override def name: String = "FLOAT" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = Some(7) + + override def javaSQLType: Int = java.sql.Types.FLOAT + + override def isNumericType(): Boolean = true + } + + case object DECIMAL extends Type { + override def name: String = "DECIMAL" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = true + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = Some(DecimalType.MAX_PRECISION) + + override def javaSQLType: Int = java.sql.Types.DECIMAL + + override def isNumericType(): Boolean = true + } + + case object LONG extends Type { + override def name: String = "LONG" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = Some(19) + + override def javaSQLType: Int = java.sql.Types.BIGINT + + override def isNumericType(): Boolean = true + } + + case object BYTE extends Type { + override def name: String = "BYTE" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.TINYINT + } + + case object SHORT extends Type { + override def name: String = "SHORT" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = Some(5) + + override def javaSQLType: Int = java.sql.Types.SMALLINT + + override def isNumericType(): Boolean = true + } + + case object DATE extends Type { + override def name: String = "DATE" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.DATE + } + + case object TIMESTAMP extends Type { + override def name: String = "TIMESTAMP" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.TIMESTAMP + } + + case object BINARY extends Type { + override def name: String = "BINARY" + + override def isComplex: Boolean = false + + override def isQualifiedType: Boolean = false + + override def isCollectionType: Boolean = false + + override def getMaxPrecision(): Option[Int] = None + + override def javaSQLType: Int = java.sql.Types.BINARY + } + +} \ No newline at end of file diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala index 35f92547e781..9b55f3da8e46 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala @@ -145,6 +145,17 @@ private[thriftserver] class SparkSQLOperationManager() operation } + override def newGetTypeInfoOperation( + parentSession: HiveSession): GetTypeInfoOperation = synchronized { + val sqlContext = sessionToContexts.get(parentSession.getSessionHandle) + require(sqlContext != null, s"Session handle: ${parentSession.getSessionHandle} has not been" + + " initialized or had already closed.") + val operation = new SparkGetTypeInfoOperation(sqlContext, parentSession) + handleToOperation.put(operation.getHandle, operation) + logDebug(s"Created GetTypeInfoOperation with session=$parentSession.") + operation + } + def setConfMap(conf: SQLConf, confMap: java.util.Map[String, String]): Unit = { val iterator = confMap.entrySet().iterator() while (iterator.hasNext) { diff --git a/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java b/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java index 0f72071d7e7d..3e81f8afbd85 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java +++ b/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java @@ -73,7 +73,7 @@ public class GetTypeInfoOperation extends MetadataOperation { .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE, "Usually 2 or 10"); - private final RowSet rowSet; + protected final RowSet rowSet; protected GetTypeInfoOperation(HiveSession parentSession) { super(parentSession, OperationType.GET_TYPE_INFO); From 1d47dd18ce3601ad8d21500ef8ff5157a9fbb7af Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 5 Sep 2019 19:15:53 +0800 Subject: [PATCH 02/13] fix method --- .../sql/hive/thriftserver/cli/Type.scala | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala index a2ca981bc2b0..a2e883f5f887 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -23,18 +23,18 @@ import org.apache.spark.sql.types.DecimalType trait Type { - def name: String + def getName: String def isComplex: Boolean - def isPrimitiveType: Boolean = !isComplex - - def isComplexType: Boolean = isComplex - def isQualifiedType: Boolean def isCollectionType: Boolean + def isPrimitiveType: Boolean = !isComplex + + def isComplexType: Boolean = isComplex + /** * Radix for this type (typically either 2 or 10) * Null is returned for data types where this is not applicable. @@ -114,9 +114,10 @@ trait Type { */ def isUnsignedAttribute: Boolean = { if (isNumericType) { - return false + false + } else { + true } - true } /** @@ -157,8 +158,6 @@ trait Type { def javaSQLType: Int def toJavaSQLType: Int = javaSQLType - - def getName: String = name } object Type { @@ -179,7 +178,7 @@ object Type { BINARY) case object NULL extends Type() { - override def name: String = "NULL" + override def getName: String = "NULL" override def isComplex: Boolean = false @@ -193,7 +192,7 @@ object Type { } case object STRING extends Type { - override def name: String = "STRING" + override def getName: String = "STRING" override def isComplex: Boolean = false @@ -209,7 +208,7 @@ object Type { } case object INT extends Type { - override def name: String = "INT" + override def getName: String = "INT" override def isComplex: Boolean = false @@ -225,7 +224,7 @@ object Type { } case object BOOLEAN extends Type { - override def name: String = "BOOLEAN" + override def getName: String = "BOOLEAN" override def isComplex: Boolean = false @@ -239,7 +238,7 @@ object Type { } case object DOUBLE extends Type { - override def name: String = "DOUBLE" + override def getName: String = "DOUBLE" override def isComplex: Boolean = false @@ -255,7 +254,7 @@ object Type { } case object FLOAT extends Type { - override def name: String = "FLOAT" + override def getName: String = "FLOAT" override def isComplex: Boolean = false @@ -271,7 +270,7 @@ object Type { } case object DECIMAL extends Type { - override def name: String = "DECIMAL" + override def getName: String = "DECIMAL" override def isComplex: Boolean = false @@ -287,7 +286,7 @@ object Type { } case object LONG extends Type { - override def name: String = "LONG" + override def getName: String = "LONG" override def isComplex: Boolean = false @@ -303,7 +302,7 @@ object Type { } case object BYTE extends Type { - override def name: String = "BYTE" + override def getName: String = "BYTE" override def isComplex: Boolean = false @@ -317,7 +316,7 @@ object Type { } case object SHORT extends Type { - override def name: String = "SHORT" + override def getName: String = "SHORT" override def isComplex: Boolean = false @@ -333,7 +332,7 @@ object Type { } case object DATE extends Type { - override def name: String = "DATE" + override def getName: String = "DATE" override def isComplex: Boolean = false @@ -347,7 +346,7 @@ object Type { } case object TIMESTAMP extends Type { - override def name: String = "TIMESTAMP" + override def getName: String = "TIMESTAMP" override def isComplex: Boolean = false @@ -361,7 +360,7 @@ object Type { } case object BINARY extends Type { - override def name: String = "BINARY" + override def getName: String = "BINARY" override def isComplex: Boolean = false From 2ec85656d955adc7c07d615e5aa2f758819b30fd Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 5 Sep 2019 21:08:23 +0800 Subject: [PATCH 03/13] Remove ThrifserverShimUtils's toJavaSQLType --- .../thriftserver/SparkGetColumnsOperation.scala | 6 +++--- .../spark/sql/hive/thriftserver/cli/Type.scala | 17 +++++++++++++++-- .../thriftserver/ThriftserverShimUtils.scala | 2 -- .../thriftserver/ThriftserverShimUtils.scala | 2 -- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala index 89faff2f6f91..826ce413b672 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala @@ -24,7 +24,7 @@ import scala.collection.JavaConverters.seqAsJavaListConverter import org.apache.hadoop.hive.ql.security.authorization.plugin.{HiveOperationType, HivePrivilegeObject} import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType -import org.apache.hive.service.cli._ +import org.apache.hive.service.cli.{CLIServiceUtils, HiveSQLException, OperationState} import org.apache.hive.service.cli.operation.GetColumnsOperation import org.apache.hive.service.cli.session.HiveSession @@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.SessionCatalog -import org.apache.spark.sql.hive.thriftserver.ThriftserverShimUtils.toJavaSQLType +import org.apache.spark.sql.hive.thriftserver.cli.Type import org.apache.spark.sql.types.StructType import org.apache.spark.util.{Utils => SparkUtils} @@ -151,7 +151,7 @@ private[hive] class SparkGetColumnsOperation( dbName, // TABLE_SCHEM tableName, // TABLE_NAME column.name, // COLUMN_NAME - toJavaSQLType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE + Type.getType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE column.dataType.sql, // TYPE_NAME null, // COLUMN_SIZE null, // BUFFER_LENGTH, unused diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala index a2e883f5f887..f1a8a8aa5976 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -165,7 +165,7 @@ object Type { def types: Seq[Type] = Seq(NULL, STRING, - INT, + INTEGER, BOOLEAN, DOUBLE, FLOAT, @@ -177,6 +177,19 @@ object Type { TIMESTAMP, BINARY) + def getType(name: String): Type = { + val typeForName: Option[Type] = types.find(_.getName == name) + if (typeForName.isDefined) { + typeForName.get + } else { + if (Array("ARRAY", "STRUCT", "MAP", "USERDEFINED").contains(name)) { + STRING + } else { + throw new IllegalArgumentException("Unrecognized type name: " + name) + } + } + } + case object NULL extends Type() { override def getName: String = "NULL" @@ -207,7 +220,7 @@ object Type { override def isCaseSensitive: Boolean = true } - case object INT extends Type { + case object INTEGER extends Type { override def getName: String = "INT" override def isComplex: Boolean = false diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 87c0f8f6a571..e2224c8e1872 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -49,8 +49,6 @@ private[thriftserver] object ThriftserverShimUtils { RowSetFactory.create(getResultSetSchema, getProtocolVersion) } - private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType - private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 124c9937c0fc..7c36be841a53 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -54,8 +54,6 @@ private[thriftserver] object ThriftserverShimUtils { RowSetFactory.create(getResultSetSchema, getProtocolVersion, false) } - private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType - private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { From d20262884bfaffb29d1489515dca3dda02a60b5c Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 5 Sep 2019 21:37:36 +0800 Subject: [PATCH 04/13] Revert "Remove ThrifserverShimUtils's toJavaSQLType" This reverts commit 2ec85656d955adc7c07d615e5aa2f758819b30fd. --- .../thriftserver/SparkGetColumnsOperation.scala | 6 +++--- .../spark/sql/hive/thriftserver/cli/Type.scala | 17 ++--------------- .../thriftserver/ThriftserverShimUtils.scala | 2 ++ .../thriftserver/ThriftserverShimUtils.scala | 2 ++ 4 files changed, 9 insertions(+), 18 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala index 826ce413b672..89faff2f6f91 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala @@ -24,7 +24,7 @@ import scala.collection.JavaConverters.seqAsJavaListConverter import org.apache.hadoop.hive.ql.security.authorization.plugin.{HiveOperationType, HivePrivilegeObject} import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType -import org.apache.hive.service.cli.{CLIServiceUtils, HiveSQLException, OperationState} +import org.apache.hive.service.cli._ import org.apache.hive.service.cli.operation.GetColumnsOperation import org.apache.hive.service.cli.session.HiveSession @@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.SessionCatalog -import org.apache.spark.sql.hive.thriftserver.cli.Type +import org.apache.spark.sql.hive.thriftserver.ThriftserverShimUtils.toJavaSQLType import org.apache.spark.sql.types.StructType import org.apache.spark.util.{Utils => SparkUtils} @@ -151,7 +151,7 @@ private[hive] class SparkGetColumnsOperation( dbName, // TABLE_SCHEM tableName, // TABLE_NAME column.name, // COLUMN_NAME - Type.getType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE + toJavaSQLType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE column.dataType.sql, // TYPE_NAME null, // COLUMN_SIZE null, // BUFFER_LENGTH, unused diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala index f1a8a8aa5976..a2e883f5f887 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -165,7 +165,7 @@ object Type { def types: Seq[Type] = Seq(NULL, STRING, - INTEGER, + INT, BOOLEAN, DOUBLE, FLOAT, @@ -177,19 +177,6 @@ object Type { TIMESTAMP, BINARY) - def getType(name: String): Type = { - val typeForName: Option[Type] = types.find(_.getName == name) - if (typeForName.isDefined) { - typeForName.get - } else { - if (Array("ARRAY", "STRUCT", "MAP", "USERDEFINED").contains(name)) { - STRING - } else { - throw new IllegalArgumentException("Unrecognized type name: " + name) - } - } - } - case object NULL extends Type() { override def getName: String = "NULL" @@ -220,7 +207,7 @@ object Type { override def isCaseSensitive: Boolean = true } - case object INTEGER extends Type { + case object INT extends Type { override def getName: String = "INT" override def isComplex: Boolean = false diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index e2224c8e1872..87c0f8f6a571 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -49,6 +49,8 @@ private[thriftserver] object ThriftserverShimUtils { RowSetFactory.create(getResultSetSchema, getProtocolVersion) } + private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType + private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 7c36be841a53..124c9937c0fc 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -54,6 +54,8 @@ private[thriftserver] object ThriftserverShimUtils { RowSetFactory.create(getResultSetSchema, getProtocolVersion, false) } + private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType + private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { From 26de9057060d5c8e908830a9406b95440ed2429f Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 5 Sep 2019 21:38:20 +0800 Subject: [PATCH 05/13] change name INT to INTEGER --- .../org/apache/spark/sql/hive/thriftserver/cli/Type.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala index a2e883f5f887..828a24d43ba3 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -165,7 +165,7 @@ object Type { def types: Seq[Type] = Seq(NULL, STRING, - INT, + INTEGER, BOOLEAN, DOUBLE, FLOAT, @@ -207,7 +207,7 @@ object Type { override def isCaseSensitive: Boolean = true } - case object INT extends Type { + case object INTEGER extends Type { override def getName: String = "INT" override def isComplex: Boolean = false From 945074de7ca112b6aa9756eb9f4f938deb0ecee3 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 5 Sep 2019 22:04:30 +0800 Subject: [PATCH 06/13] change name of INTEGER --- .../scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala index 828a24d43ba3..bd0ca8430192 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -208,7 +208,7 @@ object Type { } case object INTEGER extends Type { - override def getName: String = "INT" + override def getName: String = "INTEGER" override def isComplex: Boolean = false From d3d2efcb62f1d931f6a004df809f24bfed2d8c10 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Fri, 6 Sep 2019 17:30:05 +0800 Subject: [PATCH 07/13] change method name --- .../SparkGetTypeInfoOperation.scala | 2 +- .../sql/hive/thriftserver/cli/Type.scala | 31 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala index 215475f2524f..cf5f50abee2b 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala @@ -68,7 +68,7 @@ private[hive] class SparkGetTypeInfoOperation( parentSession.getUsername) try { - Type.types.foreach(typeInfo => { + Type.values.foreach(typeInfo => { val rowData = Array[AnyRef]( typeInfo.getName, // TYPE_NAME typeInfo.toJavaSQLType.asInstanceOf[AnyRef], // DATA_TYPE diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala index bd0ca8430192..a434ccf0167f 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala @@ -162,20 +162,23 @@ trait Type { object Type { - def types: Seq[Type] = - Seq(NULL, - STRING, - INTEGER, - BOOLEAN, - DOUBLE, - FLOAT, - DECIMAL, - LONG, - BYTE, - SHORT, - DATE, - TIMESTAMP, - BINARY) + val typeMap: Map[String, Type] = Map( + NULL.getName -> NULL, + STRING.getName -> STRING, + INTEGER.getName -> INTEGER, + BOOLEAN.getName -> BOOLEAN, + DOUBLE.getName -> DOUBLE, + FLOAT.getName -> FLOAT, + DECIMAL.getName -> DECIMAL, + LONG.getName -> LONG, + BYTE.getName -> BYTE, + SHORT.getName -> SHORT, + DATE.getName -> DATE, + TIMESTAMP.getName -> TIMESTAMP, + BINARY.getName -> BINARY + ) + + def values: Seq[Type] = typeMap.values.toSeq case object NULL extends Type() { override def getName: String = "NULL" From aa1d7213b67dba4161ebd51f3cd7ec519acd4d9d Mon Sep 17 00:00:00 2001 From: angerszhu Date: Fri, 6 Sep 2019 22:22:46 +0800 Subject: [PATCH 08/13] Use ThriftserverShimUtils to show support Type --- .../SparkGetTypeInfoOperation.scala | 5 +- .../sql/hive/thriftserver/cli/Type.scala | 379 ------------------ .../thriftserver/ThriftserverShimUtils.scala | 6 + .../cli/operation/GetTypeInfoOperation.java | 2 +- .../thriftserver/ThriftserverShimUtils.scala | 5 + 5 files changed, 14 insertions(+), 383 deletions(-) delete mode 100644 sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala index cf5f50abee2b..7a6a8c59b721 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala @@ -26,7 +26,6 @@ import org.apache.hive.service.cli.session.HiveSession import org.apache.spark.internal.Logging import org.apache.spark.sql.SQLContext -import org.apache.spark.sql.hive.thriftserver.cli.Type import org.apache.spark.util.{Utils => SparkUtils} /** @@ -68,11 +67,11 @@ private[hive] class SparkGetTypeInfoOperation( parentSession.getUsername) try { - Type.values.foreach(typeInfo => { + ThriftserverShimUtils.supportedType().foreach(typeInfo => { val rowData = Array[AnyRef]( typeInfo.getName, // TYPE_NAME typeInfo.toJavaSQLType.asInstanceOf[AnyRef], // DATA_TYPE - typeInfo.getMaxPrecision.getOrElse(null).asInstanceOf[AnyRef], // PRECISION + typeInfo.getMaxPrecision.asInstanceOf[AnyRef], // PRECISION typeInfo.getLiteralPrefix, // LITERAL_PREFIX typeInfo.getLiteralSuffix, // LITERAL_SUFFIX typeInfo.getCreateParams, // CREATE_PARAMS diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala deleted file mode 100644 index a434ccf0167f..000000000000 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.thriftserver.cli - -import java.sql.DatabaseMetaData - -import org.apache.spark.sql.types.DecimalType - -trait Type { - - def getName: String - - def isComplex: Boolean - - def isQualifiedType: Boolean - - def isCollectionType: Boolean - - def isPrimitiveType: Boolean = !isComplex - - def isComplexType: Boolean = isComplex - - /** - * Radix for this type (typically either 2 or 10) - * Null is returned for data types where this is not applicable. - */ - def getNumPrecRadix: Integer = { - if (this.isNumericType) { - 10 - } else { - null - } - } - - def getMaxPrecision(): Option[Int] - - def isNumericType(): Boolean = false - - /** - * Prefix used to quote a literal of this type (may be null) - */ - def getLiteralPrefix: String = null - - /** - * Suffix used to quote a literal of this type (may be null) - * - * @return - */ - def getLiteralSuffix: String = null - - /** - * Can you use NULL for this type? - * - * @return - * DatabaseMetaData.typeNoNulls - does not allow NULL values - * DatabaseMetaData.typeNullable - allows NULL values - * DatabaseMetaData.typeNullableUnknown - nullability unknown - */ - def getNullable: Short = { - // All Hive types are nullable - DatabaseMetaData.typeNullable.toShort - } - - /** - * Is the type case sensitive? - * - * @return - */ - def isCaseSensitive: Boolean = false - - /** - * Parameters used in creating the type (may be null) - * - * @return - */ - def getCreateParams: String = null - - /** - * Can you use WHERE based on this type? - * - * @return - * DatabaseMetaData.typePredNone - No support - * DatabaseMetaData.typePredChar - Only support with WHERE .. LIKE - * DatabaseMetaData.typePredBasic - Supported except for WHERE .. LIKE - * DatabaseMetaData.typeSearchable - Supported for all WHERE .. - */ - def getSearchable: Short = { - if (isPrimitiveType) { - DatabaseMetaData.typeSearchable.toShort - } else { - DatabaseMetaData.typePredNone.toShort - } - } - - /** - * Is this type unsigned? - * - * @return - */ - def isUnsignedAttribute: Boolean = { - if (isNumericType) { - false - } else { - true - } - } - - /** - * Can this type represent money? - * - * @return - */ - def isFixedPrecScale: Boolean = false - - /** - * Can this type be used for an auto-increment value? - * - * @return - */ - def isAutoIncrement: Boolean = false - - /** - * Localized version of type name (may be null). - * - * @return - */ - def getLocalizedName: String = null - - /** - * Minimum scale supported for this type - * - * @return - */ - def getMinimumScale: Short = 0.toShort - - /** - * Maximum scale supported for this type - * - * @return - */ - def getMaximumScale: Short = 0.toShort - - def javaSQLType: Int - - def toJavaSQLType: Int = javaSQLType -} - -object Type { - - val typeMap: Map[String, Type] = Map( - NULL.getName -> NULL, - STRING.getName -> STRING, - INTEGER.getName -> INTEGER, - BOOLEAN.getName -> BOOLEAN, - DOUBLE.getName -> DOUBLE, - FLOAT.getName -> FLOAT, - DECIMAL.getName -> DECIMAL, - LONG.getName -> LONG, - BYTE.getName -> BYTE, - SHORT.getName -> SHORT, - DATE.getName -> DATE, - TIMESTAMP.getName -> TIMESTAMP, - BINARY.getName -> BINARY - ) - - def values: Seq[Type] = typeMap.values.toSeq - - case object NULL extends Type() { - override def getName: String = "NULL" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.NULL - } - - case object STRING extends Type { - override def getName: String = "STRING" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.VARCHAR - - override def isCaseSensitive: Boolean = true - } - - case object INTEGER extends Type { - override def getName: String = "INTEGER" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = Some(10) - - override def javaSQLType: Int = java.sql.Types.INTEGER - - override def isNumericType(): Boolean = true - } - - case object BOOLEAN extends Type { - override def getName: String = "BOOLEAN" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.BOOLEAN - } - - case object DOUBLE extends Type { - override def getName: String = "DOUBLE" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = Some(15) - - override def javaSQLType: Int = java.sql.Types.DOUBLE - - override def isNumericType(): Boolean = true - } - - case object FLOAT extends Type { - override def getName: String = "FLOAT" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = Some(7) - - override def javaSQLType: Int = java.sql.Types.FLOAT - - override def isNumericType(): Boolean = true - } - - case object DECIMAL extends Type { - override def getName: String = "DECIMAL" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = true - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = Some(DecimalType.MAX_PRECISION) - - override def javaSQLType: Int = java.sql.Types.DECIMAL - - override def isNumericType(): Boolean = true - } - - case object LONG extends Type { - override def getName: String = "LONG" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = Some(19) - - override def javaSQLType: Int = java.sql.Types.BIGINT - - override def isNumericType(): Boolean = true - } - - case object BYTE extends Type { - override def getName: String = "BYTE" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.TINYINT - } - - case object SHORT extends Type { - override def getName: String = "SHORT" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = Some(5) - - override def javaSQLType: Int = java.sql.Types.SMALLINT - - override def isNumericType(): Boolean = true - } - - case object DATE extends Type { - override def getName: String = "DATE" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.DATE - } - - case object TIMESTAMP extends Type { - override def getName: String = "TIMESTAMP" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.TIMESTAMP - } - - case object BINARY extends Type { - override def getName: String = "BINARY" - - override def isComplex: Boolean = false - - override def isQualifiedType: Boolean = false - - override def isCollectionType: Boolean = false - - override def getMaxPrecision(): Option[Int] = None - - override def javaSQLType: Int = java.sql.Types.BINARY - } - -} \ No newline at end of file diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 87c0f8f6a571..2abc879f8ff1 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -51,6 +51,12 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType + private[thriftserver] def supportedType(): Seq[Type] = { + Array(Type.NULL_TYPE, Type.BOOLEAN_TYPE, Type.TINYINT_TYPE, Type.SMALLINT_TYPE, Type.INT_TYPE, + Type.BIGINT_TYPE, Type.FLOAT_TYPE, Type.DOUBLE_TYPE, Type.STRING_TYPE, Type.DATE_TYPE, + Type.TIMESTAMP_TYPE, Type.DECIMAL_TYPE, Type.BINARY_TYPE) + } + private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { diff --git a/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java b/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java index 9612eb145638..0f57a72e2a1c 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java +++ b/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java @@ -73,7 +73,7 @@ public class GetTypeInfoOperation extends MetadataOperation { .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE, "Usually 2 or 10"); - private final RowSet rowSet; + protected final RowSet rowSet; protected GetTypeInfoOperation(HiveSession parentSession) { super(parentSession, OperationType.GET_TYPE_INFO); diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 124c9937c0fc..746a0f9d053e 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -56,6 +56,11 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType + private[thriftserver] def supportedType(): Seq[Type] = { + Array(Type.NULL_TYPE, Type.BOOLEAN_TYPE, Type.TINYINT_TYPE, Type.SMALLINT_TYPE, Type.INT_TYPE, + Type.BIGINT_TYPE, Type.FLOAT_TYPE, Type.DOUBLE_TYPE, Type.STRING_TYPE, Type.DATE_TYPE, + Type.TIMESTAMP_TYPE, Type.DECIMAL_TYPE, Type.BINARY_TYPE) + } private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { From 9ee38ebe394240be371511fc75551800d5d4e5da Mon Sep 17 00:00:00 2001 From: angerszhu Date: Sat, 7 Sep 2019 08:40:48 +0800 Subject: [PATCH 09/13] Add empty line between function --- .../spark/sql/hive/thriftserver/ThriftserverShimUtils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 746a0f9d053e..7f6f462bf46a 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -61,6 +61,7 @@ private[thriftserver] object ThriftserverShimUtils { Type.BIGINT_TYPE, Type.FLOAT_TYPE, Type.DOUBLE_TYPE, Type.STRING_TYPE, Type.DATE_TYPE, Type.TIMESTAMP_TYPE, Type.DECIMAL_TYPE, Type.BINARY_TYPE) } + private[thriftserver] def addToClassPath( loader: ClassLoader, auxJars: Array[String]): ClassLoader = { From 64ad1f36f6533044dea7cecceb96d36c5cdc9c4e Mon Sep 17 00:00:00 2001 From: angerszhu Date: Sun, 8 Sep 2019 09:23:33 +0800 Subject: [PATCH 10/13] add UT --- .../HiveThriftServer2Suites.scala | 29 +++++++++++++++++-- .../thriftserver/ThriftserverShimUtils.scala | 8 +++-- .../thriftserver/ThriftserverShimUtils.scala | 8 +++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index b7185db2f2ae..ed71e8299c17 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.thriftserver import java.io.{File, FilenameFilter} import java.net.URL import java.nio.charset.StandardCharsets -import java.sql.{Date, DriverManager, SQLException, Statement} +import java.sql.{Date, DriverManager, ResultSet, SQLException, Statement} import java.util.{Locale, UUID} import scala.collection.mutable @@ -39,8 +39,8 @@ import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient import org.apache.thrift.protocol.TBinaryProtocol import org.apache.thrift.transport.TSocket import org.scalatest.BeforeAndAfterAll - import org.apache.spark.{SparkException, SparkFunSuite} + import org.apache.spark.internal.Logging import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.hive.test.HiveTestUtils @@ -93,6 +93,31 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } } + test("GetTypeInfo Thrift API") { +// withCLIServiceClient { client => +// val user = System.getProperty("user.name") +// val sessionHandle = client.openSession(user, "") +// val operationHandle = client.getTypeInfo(sessionHandle) +// client.fetchResults(operationHandle) +// +// +// } + + def checkResult(rs: ResultSet, typeNames: Seq[String]): Unit = { + for (i <- typeNames.indices) { + assert(rs.next()) + assert(rs.getString("TYPE_NAME") === typeNames(i)) + } + // Make sure there are no more elements + assert(!rs.next()) + } + + withJdbcStatement() { statement => + val metaData = statement.getConnection.getMetaData + checkResult(metaData.getTypeInfo, ThriftserverShimUtils.supportedType().map(_.getName)) + } + } + test("SPARK-16563 ThriftCLIService FetchResults repeat fetching result") { withCLIServiceClient { client => val user = System.getProperty("user.name") diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 2abc879f8ff1..ddcb26a5bb54 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -21,6 +21,7 @@ import org.apache.commons.logging.LogFactory import org.apache.hadoop.hive.ql.exec.Utilities import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hive.service.cli.{RowSet, RowSetFactory, TableSchema, Type} +import org.apache.hive.service.cli.Type._ import org.apache.hive.service.cli.thrift.TProtocolVersion._ /** @@ -52,9 +53,10 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType private[thriftserver] def supportedType(): Seq[Type] = { - Array(Type.NULL_TYPE, Type.BOOLEAN_TYPE, Type.TINYINT_TYPE, Type.SMALLINT_TYPE, Type.INT_TYPE, - Type.BIGINT_TYPE, Type.FLOAT_TYPE, Type.DOUBLE_TYPE, Type.STRING_TYPE, Type.DATE_TYPE, - Type.TIMESTAMP_TYPE, Type.DECIMAL_TYPE, Type.BINARY_TYPE) + Array(NULL_TYPE, BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, + BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, DATE_TYPE, + TIMESTAMP_TYPE, DECIMAL_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, + STRUCT_TYPE, USER_DEFINED_TYPE) } private[thriftserver] def addToClassPath( diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 7f6f462bf46a..274ebc25eddf 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -24,6 +24,7 @@ import scala.collection.JavaConverters._ import org.apache.hadoop.hive.ql.exec.AddToClassPathAction import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.hive.serde2.thrift.Type +import org.apache.hadoop.hive.serde2.thrift.Type._ import org.apache.hive.service.cli.{RowSet, RowSetFactory, TableSchema} import org.apache.hive.service.rpc.thrift.TProtocolVersion._ import org.slf4j.LoggerFactory @@ -57,9 +58,10 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType private[thriftserver] def supportedType(): Seq[Type] = { - Array(Type.NULL_TYPE, Type.BOOLEAN_TYPE, Type.TINYINT_TYPE, Type.SMALLINT_TYPE, Type.INT_TYPE, - Type.BIGINT_TYPE, Type.FLOAT_TYPE, Type.DOUBLE_TYPE, Type.STRING_TYPE, Type.DATE_TYPE, - Type.TIMESTAMP_TYPE, Type.DECIMAL_TYPE, Type.BINARY_TYPE) + Array(NULL_TYPE, BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, + BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, DATE_TYPE, + TIMESTAMP_TYPE, DECIMAL_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, + STRUCT_TYPE, USER_DEFINED_TYPE) } private[thriftserver] def addToClassPath( From 51cb6360e94d544856d0b880099d6ab64e3c0840 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Sun, 8 Sep 2019 10:01:33 +0800 Subject: [PATCH 11/13] add ut --- .../HiveThriftServer2Suites.scala | 29 ++----------------- .../SparkMetadataOperationSuite.scala | 16 ++++++++++ 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index ed71e8299c17..b7185db2f2ae 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.thriftserver import java.io.{File, FilenameFilter} import java.net.URL import java.nio.charset.StandardCharsets -import java.sql.{Date, DriverManager, ResultSet, SQLException, Statement} +import java.sql.{Date, DriverManager, SQLException, Statement} import java.util.{Locale, UUID} import scala.collection.mutable @@ -39,8 +39,8 @@ import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient import org.apache.thrift.protocol.TBinaryProtocol import org.apache.thrift.transport.TSocket import org.scalatest.BeforeAndAfterAll -import org.apache.spark.{SparkException, SparkFunSuite} +import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.internal.Logging import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.hive.test.HiveTestUtils @@ -93,31 +93,6 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } } - test("GetTypeInfo Thrift API") { -// withCLIServiceClient { client => -// val user = System.getProperty("user.name") -// val sessionHandle = client.openSession(user, "") -// val operationHandle = client.getTypeInfo(sessionHandle) -// client.fetchResults(operationHandle) -// -// -// } - - def checkResult(rs: ResultSet, typeNames: Seq[String]): Unit = { - for (i <- typeNames.indices) { - assert(rs.next()) - assert(rs.getString("TYPE_NAME") === typeNames(i)) - } - // Make sure there are no more elements - assert(!rs.next()) - } - - withJdbcStatement() { statement => - val metaData = statement.getConnection.getMetaData - checkResult(metaData.getTypeInfo, ThriftserverShimUtils.supportedType().map(_.getName)) - } - } - test("SPARK-16563 ThriftCLIService FetchResults repeat fetching result") { withCLIServiceClient { client => val user = System.getProperty("user.name") diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index 21870ffd463e..f7ee3e0a46cd 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -231,4 +231,20 @@ class SparkMetadataOperationSuite extends HiveThriftJdbcTest { assert(!rs.next()) } } + + test("GetTypeInfo Thrift API") { + def checkResult(rs: ResultSet, typeNames: Seq[String]): Unit = { + for (i <- typeNames.indices) { + assert(rs.next()) + assert(rs.getString("TYPE_NAME") === typeNames(i)) + } + // Make sure there are no more elements + assert(!rs.next()) + } + + withJdbcStatement() { statement => + val metaData = statement.getConnection.getMetaData + checkResult(metaData.getTypeInfo, ThriftserverShimUtils.supportedType().map(_.getName)) + } + } } From 068352e57ca4a3ec3aa65841b26d5a5a1aad431f Mon Sep 17 00:00:00 2001 From: angerszhu Date: Sun, 8 Sep 2019 11:13:23 +0800 Subject: [PATCH 12/13] change Type order --- .../sql/hive/thriftserver/ThriftserverShimUtils.scala | 9 +++++---- .../sql/hive/thriftserver/ThriftserverShimUtils.scala | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index ddcb26a5bb54..e423f03d269c 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -53,10 +53,11 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType private[thriftserver] def supportedType(): Seq[Type] = { - Array(NULL_TYPE, BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, - BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, DATE_TYPE, - TIMESTAMP_TYPE, DECIMAL_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, - STRUCT_TYPE, USER_DEFINED_TYPE) + Seq(NULL_TYPE, BOOLEAN_TYPE, STRING_TYPE, BINARY_TYPE, + TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, + FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE, + DATE_TYPE, TIMESTAMP_TYPE, + ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, USER_DEFINED_TYPE) } private[thriftserver] def addToClassPath( diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 274ebc25eddf..372bfb3c4847 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -58,10 +58,11 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType private[thriftserver] def supportedType(): Seq[Type] = { - Array(NULL_TYPE, BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, - BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, DATE_TYPE, - TIMESTAMP_TYPE, DECIMAL_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, - STRUCT_TYPE, USER_DEFINED_TYPE) + Seq(NULL_TYPE, BOOLEAN_TYPE, STRING_TYPE, BINARY_TYPE, + TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, + FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE, + DATE_TYPE, TIMESTAMP_TYPE, + ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, USER_DEFINED_TYPE) } private[thriftserver] def addToClassPath( From 35c5a806b46e5180da56170ebb1f44a8efa8949f Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 10 Sep 2019 17:49:00 +0800 Subject: [PATCH 13/13] remove USER_DEFINED_TYPE --- .../spark/sql/hive/thriftserver/ThriftserverShimUtils.scala | 2 +- .../spark/sql/hive/thriftserver/ThriftserverShimUtils.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index e423f03d269c..837861a77bf5 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -57,7 +57,7 @@ private[thriftserver] object ThriftserverShimUtils { TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE, DATE_TYPE, TIMESTAMP_TYPE, - ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, USER_DEFINED_TYPE) + ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE) } private[thriftserver] def addToClassPath( diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 372bfb3c4847..cb32ebed0ac1 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -62,7 +62,7 @@ private[thriftserver] object ThriftserverShimUtils { TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, DECIMAL_TYPE, DATE_TYPE, TIMESTAMP_TYPE, - ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, USER_DEFINED_TYPE) + ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE) } private[thriftserver] def addToClassPath(