Skip to content

[SPARK-28982][SQL] Implementation Spark's own GetTypeInfoOperation #25694

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AngersZhuuuu wants to merge 15 commits into from
Closed

[SPARK-28982][SQL] Implementation Spark's own GetTypeInfoOperation #25694

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f61d7c2
add spark's type
AngersZhuuuu Sep 5, 2019
1d47dd1
fix method
AngersZhuuuu Sep 5, 2019
2ec8565
Remove ThrifserverShimUtils's toJavaSQLType
AngersZhuuuu Sep 5, 2019
d202628
Revert "Remove ThrifserverShimUtils's toJavaSQLType"
AngersZhuuuu Sep 5, 2019
26de905
change name INT to INTEGER
AngersZhuuuu Sep 5, 2019
945074d
change name of INTEGER
AngersZhuuuu Sep 5, 2019
d3d2efc
change method name
AngersZhuuuu Sep 6, 2019
aa1d721
Use ThriftserverShimUtils to show support Type
AngersZhuuuu Sep 6, 2019
9ee38eb
Add empty line between function
AngersZhuuuu Sep 7, 2019
64ad1f3
add UT
AngersZhuuuu Sep 8, 2019
51cb636
add ut
AngersZhuuuu Sep 8, 2019
068352e
change Type order
AngersZhuuuu Sep 8, 2019
1c919a0
Merge branch 'master' into SPARK-28982
AngersZhuuuu Sep 10, 2019
35c5a80
remove USER_DEFINED_TYPE
AngersZhuuuu Sep 10, 2019
3c488fc
Merge branch 'SPARK-28982' of https://github.com/AngersZhuuuu/spark i…
AngersZhuuuu Sep 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove ThrifserverShimUtils's toJavaSQLType
  • Loading branch information
@AngersZhuuuu
AngersZhuuuu committed Sep 5, 2019
commit 2ec85656d955adc7c07d615e5aa2f758819b30fd
6 changes: 3 additions & 3 deletions ...rver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ import scala.collection.JavaConverters.seqAsJavaListConverter

import org.apache.hadoop.hive.ql.security.authorization.plugin.{HiveOperationType, HivePrivilegeObject}
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType
import org.apache.hive.service.cli._
import org.apache.hive.service.cli.{CLIServiceUtils, HiveSQLException, OperationState}
import org.apache.hive.service.cli.operation.GetColumnsOperation
import org.apache.hive.service.cli.session.HiveSession

import org.apache.spark.internal.Logging
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.SessionCatalog
import org.apache.spark.sql.hive.thriftserver.ThriftserverShimUtils.toJavaSQLType
import org.apache.spark.sql.hive.thriftserver.cli.Type
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.{Utils => SparkUtils}

Expand Down Expand Up @@ -151,7 +151,7 @@ private[hive] class SparkGetColumnsOperation(
dbName, // TABLE_SCHEM
tableName, // TABLE_NAME
column.name, // COLUMN_NAME
toJavaSQLType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE
Type.getType(column.dataType.sql).asInstanceOf[AnyRef], // DATA_TYPE
column.dataType.sql, // TYPE_NAME
null, // COLUMN_SIZE
null, // BUFFER_LENGTH, unused
Expand Down
17 changes: 15 additions & 2 deletions sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/cli/Type.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ object Type {
def types: Seq[Type] =
Seq(NULL,
STRING,
INT,
INTEGER,
BOOLEAN,
DOUBLE,
FLOAT,
Expand All @@ -177,6 +177,19 @@ object Type {
TIMESTAMP,
BINARY)

def getType(name: String): Type = {
val typeForName: Option[Type] = types.find(_.getName == name)
if (typeForName.isDefined) {
typeForName.get
} else {
if (Array("ARRAY", "STRUCT", "MAP", "USERDEFINED").contains(name)) {
STRING
} else {
throw new IllegalArgumentException("Unrecognized type name: " + name)
}
}
}

case object NULL extends Type() {
override def getName: String = "NULL"

Expand Down Expand Up @@ -207,7 +220,7 @@ object Type {
override def isCaseSensitive: Boolean = true
}

case object INT extends Type {
case object INTEGER extends Type {
override def getName: String = "INT"

override def isComplex: Boolean = false
Expand Down
2 changes: 0 additions & 2 deletions .../v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,6 @@ private[thriftserver] object ThriftserverShimUtils {
RowSetFactory.create(getResultSetSchema, getProtocolVersion)
}

private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType

private[thriftserver] def addToClassPath(
loader: ClassLoader,
auxJars: Array[String]): ClassLoader = {
Expand Down
2 changes: 0 additions & 2 deletions .../v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@ private[thriftserver] object ThriftserverShimUtils {
RowSetFactory.create(getResultSetSchema, getProtocolVersion, false)
}

private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType

private[thriftserver] def addToClassPath(
Copy link
Contributor

@juliuszsompolski juliuszsompolski Sep 6, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: empty line between functions.

Copy link
Contributor Author

@AngersZhuuuu AngersZhuuuu Sep 7, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add it .

loader: ClassLoader,
auxJars: Array[String]): ClassLoader = {
Expand Down