Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
8b56400
Add new module and add v11 thrift protocol
AngersZhuuuu Oct 22, 2019
e30f686
add pom lincense
AngersZhuuuu Oct 23, 2019
4a50bc9
update code folder name
AngersZhuuuu Oct 23, 2019
7ffdc3b
all like hive
AngersZhuuuu Oct 23, 2019
b74d5e0
remove py/cpp/r/php
AngersZhuuuu Oct 23, 2019
fc2648f
Maven generate thrift source code
wangyum Oct 24, 2019
5365dcf
org.apache.thrift.tools:maven-thrift-plugin -> org.apache.thrift:thri…
wangyum Oct 25, 2019
95d8137
save some basic code
AngersZhuuuu Oct 26, 2019
4dc5c7e
Merge pull request #2 from wangyum/SPARK-29108
AngersZhuuuu Oct 27, 2019
24ce6d4
Merge branch 'SPARK-29018-V11' of https://github.com/AngersZhuuuu/spa…
AngersZhuuuu Oct 27, 2019
5efe8cb
Revert "save some basic code"
AngersZhuuuu Oct 27, 2019
6f7d48a
Update TCLIService.thrift
AngersZhuuuu Oct 27, 2019
cf24306
Add basic data
AngersZhuuuu Oct 27, 2019
24fce6b
save code
AngersZhuuuu Oct 28, 2019
bc36bdf
change
AngersZhuuuu Oct 28, 2019
c438673
format code
AngersZhuuuu Oct 28, 2019
a7ec8b0
format code
AngersZhuuuu Oct 29, 2019
5cf7eb0
Update SparkSQLEnv.scala
AngersZhuuuu Oct 29, 2019
0023bcb
fix logger conflict
AngersZhuuuu Oct 29, 2019
723450b
fix scala style
AngersZhuuuu Oct 29, 2019
3eb7672
start with execution hive
AngersZhuuuu Oct 29, 2019
5aa4d9d
format code add file header
AngersZhuuuu Oct 29, 2019
782d36b
FOR BUILD spark-thriftserver
AngersZhuuuu Oct 29, 2019
f2b5346
add UT class
AngersZhuuuu Oct 29, 2019
a14a9e9
fix UT case, remove can't supported UT
AngersZhuuuu Oct 29, 2019
e1bb6e1
fix for UT
AngersZhuuuu Oct 29, 2019
74fb240
fix UT
AngersZhuuuu Oct 29, 2019
2f90ed9
fix for UT
AngersZhuuuu Oct 30, 2019
cd92f3f
fix return error
AngersZhuuuu Oct 30, 2019
60dc24f
fix client convert row map
AngersZhuuuu Oct 30, 2019
409a1a3
add UT of TCLIServiceClient
AngersZhuuuu Oct 31, 2019
5771a9b
fix java code style
AngersZhuuuu Oct 31, 2019
77578c3
add processGlobalInitFile
AngersZhuuuu Oct 31, 2019
4f7cbac
Update ThriftServerSessionImpl.scala
AngersZhuuuu Oct 31, 2019
d93ab72
Keep type in scala and remove unused type
AngersZhuuuu Oct 31, 2019
ac9ad54
basic service to java
AngersZhuuuu Oct 31, 2019
795ada1
Handle to java
AngersZhuuuu Oct 31, 2019
63076d3
Operation Type State Status to java
AngersZhuuuu Oct 31, 2019
000443a
Fetch type and orientation to java
AngersZhuuuu Oct 31, 2019
5de2bf9
PatternOrIdentifier to java
AngersZhuuuu Oct 31, 2019
a6b0ed1
Interface to Java
AngersZhuuuu Oct 31, 2019
2e6d221
fix bug
AngersZhuuuu Nov 1, 2019
158b298
remove setApplicationName
AngersZhuuuu Nov 1, 2019
745ca60
Session Basic Class to java
AngersZhuuuu Nov 1, 2019
f62e577
auth and thrift service to java
AngersZhuuuu Nov 1, 2019
5fae842
Update ColumnBasedSet.scala
AngersZhuuuu Nov 1, 2019
fe39db0
Update RowBasedSet.scala
AngersZhuuuu Nov 1, 2019
1c303ba
Update type map
AngersZhuuuu Nov 1, 2019
9a32af8
make RowSetFactory clean
AngersZhuuuu Nov 1, 2019
287c6be
update
AngersZhuuuu Nov 2, 2019
f6a7736
format code
AngersZhuuuu Nov 2, 2019
f8b7351
add thrift file
AngersZhuuuu Nov 6, 2019
272ba3c
not check current java code now
AngersZhuuuu Nov 6, 2019
0de3191
remove import jdk.tools dependency
AngersZhuuuu Nov 6, 2019
6374f42
follow comment
AngersZhuuuu Nov 6, 2019
f464773
Merge branch 'master' into SPARK-29018-V11-STEP4-ADD-TEST
AngersZhuuuu Nov 7, 2019
13da926
fix for `mvn install`
AngersZhuuuu Nov 7, 2019
d0de49f
fix for `mvn install`
AngersZhuuuu Nov 7, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
format code
  • Loading branch information
AngersZhuuuu committed Oct 28, 2019
commit c4386731b43c84bf2508273ef5ff7c3c41de62cd
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ import org.apache.spark.sql.thriftserver.Service._
* @param name
* service name
*/
abstract class AbstractService(val name: String) extends Service with Logging {
private[thriftserver] abstract class AbstractService(val name: String)
extends Service with Logging {

/**
* Service state: initially {@link STATE#NOTINITED}.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ import org.apache.hadoop.hive.conf.HiveConf
import org.apache.spark.internal.Logging
import org.apache.spark.sql.thriftserver.Service.STOPPED

class CompositeService(name: String) extends AbstractService(name) with Logging {
private[thriftserver] class CompositeService(name: String)
extends AbstractService(name)
with Logging {


private val serviceList: util.ArrayList[Service] = new util.ArrayList[Service]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ import org.apache.spark.internal.Logging
* and appends it to the cookie value generated at the
* server side. It uses SHA digest algorithm to sign and verify signatures.
*/
object CookieSigner {
private[thriftserver] object CookieSigner {
private val SIGNATURE = "&s="
private val SHA_STRING = "SHA"
}

class CookieSigner extends Logging {
private[thriftserver] class CookieSigner extends Logging {
private var secretBytes: Array[Byte] = null

def this(secret: Array[Byte]) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.spark.sql.thriftserver

import org.apache.hadoop.hive.conf.HiveConf

trait Service {
private[thriftserver] trait Service {
/**
* Initialize the service.
*
Expand Down Expand Up @@ -98,7 +98,7 @@ trait Service {
def getStartTime: Long
}

object Service {
private[thriftserver] object Service {

trait STATE

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

package org.apache.spark.sql.thriftserver

class ServiceException(msg: String, cause: Throwable)
private[thriftserver] class ServiceException(msg: String, cause: Throwable)
extends RuntimeException(msg, cause) {

def this(cause: Throwable) = this(null, cause)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import org.apache.hadoop.hive.conf.HiveConf
import org.apache.spark.internal.Logging
import org.apache.spark.sql.thriftserver.Service._

object ServiceOperations extends Logging {
private[thriftserver] object ServiceOperations extends Logging {

/**
* Verify that a service is in a given state.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ package org.apache.spark.sql.thriftserver
* ServiceStateChangeListener.
*
*/
trait ServiceStateChangeListener {
private[thriftserver] trait ServiceStateChangeListener {

/**
* Callback to notify of a state change. The service will already
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import java.io.{Closeable, IOException}

import org.slf4j.Logger

object ServiceUtils {
private[thriftserver] object ServiceUtils {

/*
* Get the index separating the user name from domain name (the user's name up
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import org.apache.spark.util.{ShutdownHookManager, Utils}
* The main entry point for the Spark SQL port of HiveServer2. Starts up a `SparkSQLContext` and a
* `HiveThriftServer2` thrift server.
*/
object SparkThriftServer2 extends Logging {
private[thriftserver] object SparkThriftServer2 extends Logging {
var uiTab: Option[ThriftServerTab] = None
var listener: SparkThriftServerListener = _

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,9 @@ import org.apache.spark.sql.thriftserver.cli.thrift._
import org.apache.spark.sql.thriftserver.server.SparkThriftServer
import org.apache.spark.sql.types.StructType

class CLIService(hiveServer2: SparkThriftServer, sqlContext: SQLContext)
private[thriftserver] class CLIService(hiveServer2: SparkThriftServer, sqlContext: SQLContext)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what are the changes / could it stay in Java?

extends CompositeService(classOf[CLIService].getSimpleName)
with ICLIService with Logging {

import CLIService._
with ICLIService with Logging {

private var hiveConf: HiveConf = null
private var sessionManager: SessionManager = null
Expand Down Expand Up @@ -100,23 +98,7 @@ class CLIService(hiveServer2: SparkThriftServer, sqlContext: SQLContext)

def getHttpUGI: UserGroupInformation = this.httpUGI

def openSession(protocol: TProtocolVersion,
username: String,
password: String,
configuration: Predef.Map[String, String]): SessionHandle = {
val sessionHandle: SessionHandle =
sessionManager.openSession(protocol,
username,
password,
null,
configuration,
false,
null)
logDebug(sessionHandle + ": openSession()")
sessionHandle
}

def openSession(protocol: TProtocolVersion,
override def openSession(protocol: TProtocolVersion,
username: String,
password: String,
ipAddress: String,
Expand All @@ -133,24 +115,7 @@ class CLIService(hiveServer2: SparkThriftServer, sqlContext: SQLContext)
sessionHandle
}

def openSessionWithImpersonation(protocol: TProtocolVersion,
username: String,
password: String,
configuration: Predef.Map[String, String],
delegationToken: String): SessionHandle = {
val sessionHandle =
sessionManager.openSession(protocol,
username,
password,
null,
configuration,
true,
delegationToken)
logDebug(sessionHandle + ": openSessionWithImpersonation()")
sessionHandle
}

def openSessionWithImpersonation(protocol: TProtocolVersion,
override def openSessionWithImpersonation(protocol: TProtocolVersion,
username: String,
password: String,
ipAddress: String,
Expand All @@ -168,38 +133,6 @@ class CLIService(hiveServer2: SparkThriftServer, sqlContext: SQLContext)
sessionHandle
}

override def openSession(username: String,
password: String,
configuration: Predef.Map[String, String]): SessionHandle = {
val sessionHandle =
sessionManager.openSession(
SERVER_VERSION,
username,
password,
null,
configuration,
false,
null)
logDebug(sessionHandle + ": openSession()")
sessionHandle
}

override def openSessionWithImpersonation(username: String,
password: String,
configuration: Predef.Map[String, String],
delegationToken: String): SessionHandle = {
val sessionHandle =
sessionManager.openSession(SERVER_VERSION,
username,
password,
null,
configuration,
true,
delegationToken)
logDebug(sessionHandle + ": openSessionWithImpersonation()")
sessionHandle
}

override def closeSession(sessionHandle: SessionHandle): Unit = {
sessionManager.closeSession(sessionHandle)
logDebug(sessionHandle + ": closeSession()")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,10 @@ import org.apache.spark.sql.execution.HiveResult
import org.apache.spark.sql.thriftserver.cli.thrift._
import org.apache.spark.sql.types.{BinaryType, _}

case class ColumnBasedSet(types: StructType,
rows: ArrayBuffer[Row],
initStartOffset: Long)
private[thriftserver] case class ColumnBasedSet(
types: StructType,
rows: ArrayBuffer[Row],
initStartOffset: Long)
extends RowSet {

var startOffset: Long = initStartOffset
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import org.apache.spark.sql.types.StructField
* A wrapper class for Spark's [[StructField]] with a column position,
* and can be transform to [[TColumnDesc]]
*/
case class ColumnDescriptor(field: StructField, pos: Int) {
private[thriftserver] case class ColumnDescriptor(field: StructField, pos: Int) {
/**
* Transform a [[ColumnDescriptor]] to a [[TColumnDesc]] instance.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ package org.apache.spark.sql.thriftserver.cli

import org.apache.spark.sql.thriftserver.cli.thrift.TFetchOrientation

trait FetchOrientation {
private[thriftserver] trait FetchOrientation {
def toTFetchOrientation: TFetchOrientation
}

object FetchOrientation {
private[thriftserver] object FetchOrientation {

case object FETCH_NEXT extends FetchOrientation {
override val toTFetchOrientation: TFetchOrientation = TFetchOrientation.FETCH_NEXT
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@

package org.apache.spark.sql.thriftserver.cli

trait FetchType {
private[thriftserver] trait FetchType {
def toTFetchType: Short
}

object FetchType {
private[thriftserver] object FetchType {

case object QUERY_OUTPUT extends FetchType {
override val toTFetchType: Short = 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.spark.sql.thriftserver.cli
import org.apache.spark.sql.thriftserver.cli.thrift.THandleIdentifier


abstract class Handle(val handleId: HandleIdentifier) {
private[thriftserver] abstract class Handle(val handleId: HandleIdentifier) {
def this() = this(new HandleIdentifier())

def this(tHandleIdentifier: THandleIdentifier) = this(new HandleIdentifier(tHandleIdentifier))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import java.util.UUID

import org.apache.spark.sql.thriftserver.cli.thrift.THandleIdentifier

class HandleIdentifier(val publicId: UUID, val secretId: UUID) {
private[thriftserver] class HandleIdentifier(val publicId: UUID, val secretId: UUID) {

def this() = this(UUID.randomUUID(), UUID.randomUUID())

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,24 @@
package org.apache.spark.sql.thriftserver.cli

import org.apache.spark.sql.thriftserver.auth.HiveAuthFactory
import org.apache.spark.sql.thriftserver.cli.thrift.{TGetQueryIdReq, TGetQueryIdResp, TSetClientInfoReq, TSetClientInfoResp}
import org.apache.spark.sql.thriftserver.cli.thrift._
import org.apache.spark.sql.types.StructType


trait ICLIService {
private[thriftserver] trait ICLIService {

@throws[SparkThriftServerSQLException]
def openSession(username: String,
def openSession(protocol: TProtocolVersion,
username: String,
password: String,
ipAddress: String,
configuration: Map[String, String]): SessionHandle

@throws[SparkThriftServerSQLException]
def openSessionWithImpersonation(username: String,
def openSessionWithImpersonation(protocol: TProtocolVersion,
username: String,
password: String,
ipAddress: String,
configuration: Map[String, String],
delegationToken: String): SessionHandle

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ package org.apache.spark.sql.thriftserver.cli

import org.apache.spark.sql.thriftserver.cli.thrift.{TOperationHandle, TProtocolVersion}

class OperationHandle private(val opType: OperationType,
protocol: TProtocolVersion,
handleId: HandleIdentifier) extends Handle(handleId) {
private[thriftserver] class OperationHandle private(
val opType: OperationType,
protocol: TProtocolVersion,
handleId: HandleIdentifier) extends Handle(handleId) {

private[this] var hasResultSet: Boolean = false

Expand Down
Loading