Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'upstream/master' into add_date_timestamp
Conflicts:
	sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
	sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala
  • Loading branch information
viirya committed Jun 13, 2015
commit fb532b5af87d2c2edc440a383397c28081067619
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import scala.collection.mutable.ArraySeq;

import org.apache.spark.sql.catalyst.util.DateUtils;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.BaseMutableRow;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.StructType;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.spark.sql.catalyst.util.DateUtils
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.PlatformDependent
import org.apache.spark.unsafe.array.ByteArrayMethods
Expand Down Expand Up @@ -265,24 +266,24 @@ private class StringUnsafeColumnWriter private() extends UnsafeColumnWriter {
}

private class DateUnsafeColumnWriter private() extends UnsafeColumnWriter {
def getSize(source: Row, column: Int): Int = {
def getSize(source: InternalRow, column: Int): Int = {
0
}

override def write(source: Row, target: UnsafeRow, column: Int, appendCursor: Int): Int = {
override def write(source: InternalRow, target: UnsafeRow, column: Int, appendCursor: Int): Int = {
target.setDate(column, source.getDate(column))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should use target.setInt(column, source.getInt(column))

0
}
}

private class TimestampUnsafeColumnWriter private() extends UnsafeColumnWriter {
def getSize(source: Row, column: Int): Int = {
def getSize(source: InternalRow, column: Int): Int = {
// Although Timestamp is fixed length, it needs 12-byte that is more than 8-byte word per field.
// So we need to store it at the variable-length section.
16
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use use Long for TimestampType in catalyst now (with precision of 100ns)

}

override def write(source: Row, target: UnsafeRow, column: Int, appendCursor: Int): Int = {
override def write(source: InternalRow, target: UnsafeRow, column: Int, appendCursor: Int): Int = {
val value = DateUtils.toJavaTimestamp(source.get(column).asInstanceOf[Long])
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

target.setLong(column, source.getLong(column))

val time = value.getTime()
val nanos = value.getNanos()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ package org.apache.spark.sql.catalyst.expressions
import java.sql.{Date, Timestamp}

import org.apache.spark.sql.catalyst.util.DateUtils
import org.apache.spark.sql.types.{UTF8String, DataType, StructType, AtomicType}
import org.apache.spark.sql.types.{DataType, StructType, AtomicType}
import org.apache.spark.unsafe.types.UTF8String

/**
* An extended interface to [[InternalRow]] that allows the values for each column to be updated.
Expand Down Expand Up @@ -219,7 +220,7 @@ class GenericMutableRow(v: Array[Any]) extends GenericRow(v) with MutableRow {
override def setInt(ordinal: Int, value: Int): Unit = { values(ordinal) = value }
override def setLong(ordinal: Int, value: Long): Unit = { values(ordinal) = value }
override def setString(ordinal: Int, value: String): Unit = {
values(ordinal) = UTF8String(value)
values(ordinal) = UTF8String.fromString(value)
}

override def setDate(ordinal: Int, value: java.sql.Date): Unit = {
Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.