Skip to content

Commit 99e61fb

Browse files
marmbrusmarmbrus
committed
Merge pull request apache#51 from marmbrus/expressionEval
Several performance enhancements and simplifications of the expression evaluation framework.
2 parents 608a29e + da9afbd commit 99e61fb

35 files changed

+939
-552
lines changed

‎project/SparkBuild.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import AssemblyKeys._
2323
import scala.util.Properties
2424
import org.scalastyle.sbt.ScalastylePlugin.{Settings => ScalaStyleSettings}
2525

26+
import scala.collection.JavaConversions._
27+
2628
// For Sonatype publishing
2729
//import com.jsuereth.pgp.sbtplugin.PgpKeys._
2830

@@ -144,6 +146,7 @@ object SparkBuild extends Build {
144146
fork := true,
145147
javaOptions in Test += "-Dspark.home=" + sparkHome,
146148
javaOptions in Test += "-Dspark.testing=1",
149+
javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark").map { case (k,v) => s"-D$k=$v" }.toSeq,
147150
javaOptions += "-Xmx3g",
148151
// Show full stack trace and duration in test cases.
149152
testOptions in Test += Tests.Argument("-oDF"),

‎sbt/sbt‎

Lines changed: 102 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,102 @@
1-
#!/bin/bash
2-
3-
#
4-
# Licensed to the Apache Software Foundation (ASF) under one or more
5-
# contributor license agreements. See the NOTICE file distributed with
6-
# this work for additional information regarding copyright ownership.
7-
# The ASF licenses this file to You under the Apache License, Version 2.0
8-
# (the "License"); you may not use this file except in compliance with
9-
# the License. You may obtain a copy of the License at
10-
#
11-
# http://www.apache.org/licenses/LICENSE-2.0
12-
#
13-
# Unless required by applicable law or agreed to in writing, software
14-
# distributed under the License is distributed on an "AS IS" BASIS,
15-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16-
# See the License for the specific language governing permissions and
17-
# limitations under the License.
18-
#
19-
20-
# Required to correctly launch hadoop/hive for generating the golden answers.
21-
HADOOP_CLASSPATH=""
22-
23-
for i in $HIVE_HOME/lib/*
24-
do HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$i
25-
done
26-
27-
export HADOOP_CLASSPATH
28-
29-
# This script launches sbt for this project. If present it uses the system
30-
# version of sbt. If there is no system version of sbt it attempts to download
31-
# sbt locally.
32-
SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
33-
URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
34-
URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
35-
JAR=sbt/sbt-launch-${SBT_VERSION}.jar
36-
37-
# Download sbt launch jar if it hasn't been downloaded yet
38-
if [ ! -f ${JAR} ]; then
39-
# Download
40-
printf "Attempting to fetch sbt\n"
41-
JAR_DL=${JAR}.part
42-
if hash curl 2>/dev/null; then
43-
(curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
44-
elif hash wget 2>/dev/null; then
45-
(wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
46-
else
47-
printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
48-
exit -1
49-
fi
50-
fi
51-
if [ ! -f ${JAR} ]; then
52-
# We failed to download
53-
printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
54-
exit -1
55-
fi
56-
printf "Launching sbt from ${JAR}\n"
57-
java \
58-
-Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=256m \
59-
-jar ${JAR} \
60-
"$@"
1+
#!/usr/bin/env bash
2+
3+
realpath () {
4+
(
5+
TARGET_FILE=$1
6+
7+
cd $(dirname $TARGET_FILE)
8+
TARGET_FILE=$(basename $TARGET_FILE)
9+
10+
COUNT=0
11+
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
12+
do
13+
TARGET_FILE=$(readlink $TARGET_FILE)
14+
cd $(dirname $TARGET_FILE)
15+
TARGET_FILE=$(basename $TARGET_FILE)
16+
COUNT=$(($COUNT + 1))
17+
done
18+
19+
echo $(pwd -P)/$TARGET_FILE
20+
)
21+
}
22+
23+
. $(dirname $(realpath $0))/sbt-launch-lib.bash
24+
25+
26+
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
27+
declare -r sbt_opts_file=".sbtopts"
28+
declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
29+
30+
usage() {
31+
cat <<EOM
32+
Usage: $script_name [options]
33+
34+
-h | -help print this message
35+
-v | -verbose this runner is chattier
36+
-d | -debug set sbt log level to debug
37+
-no-colors disable ANSI color codes
38+
-sbt-create start sbt even if current directory contains no sbt project
39+
-sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt)
40+
-sbt-boot <path> path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
41+
-ivy <path> path to local Ivy repository (default: ~/.ivy2)
42+
-mem <integer> set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
43+
-no-share use all local caches; no sharing
44+
-no-global uses global caches, but does not use global ~/.sbt directory.
45+
-jvm-debug <port> Turn on JVM debugging, open at the given port.
46+
-batch Disable interactive mode
47+
48+
# sbt version (default: from project/build.properties if present, else latest release)
49+
-sbt-version <version> use the specified version of sbt
50+
-sbt-jar <path> use the specified jar as the sbt launcher
51+
-sbt-rc use an RC version of sbt
52+
-sbt-snapshot use a snapshot version of sbt
53+
54+
# java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
55+
-java-home <path> alternate JAVA_HOME
56+
57+
# jvm options and output control
58+
JAVA_OPTS environment variable, if unset uses "$java_opts"
59+
SBT_OPTS environment variable, if unset uses "$default_sbt_opts"
60+
.sbtopts if this file exists in the current directory, it is
61+
prepended to the runner args
62+
/etc/sbt/sbtopts if this file exists, it is prepended to the runner args
63+
-Dkey=val pass -Dkey=val directly to the java runtime
64+
-J-X pass option -X directly to the java runtime
65+
(-J is stripped)
66+
-S-X add -X to sbt's scalacOptions (-J is stripped)
67+
68+
In the case of duplicated or conflicting options, the order above
69+
shows precedence: JAVA_OPTS lowest, command line options highest.
70+
EOM
71+
}
72+
73+
process_my_args () {
74+
while [[ $# -gt 0 ]]; do
75+
case "$1" in
76+
-no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
77+
-no-share) addJava "$noshare_opts" && shift ;;
78+
-no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;;
79+
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
80+
-sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;;
81+
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
82+
-batch) exec </dev/null && shift ;;
83+
84+
-sbt-create) sbt_create=true && shift ;;
85+
86+
*) addResidual "$1" && shift ;;
87+
esac
88+
done
89+
90+
# Now, ensure sbt version is used.
91+
[[ "${sbt_version}XXX" != "XXX" ]] && addJava "-Dsbt.version=$sbt_version"
92+
}
93+
94+
loadConfigFile() {
95+
cat "$1" | sed '/^\#/d'
96+
}
97+
98+
# if sbtopts files exist, prepend their contents to $@ so it can be processed by this runner
99+
[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
100+
[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
101+
102+
run "$@"

‎sbt/sbt-launch-lib.bash‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
#!/usr/bin/env bash
2+
#
3+
4+
# A library to simplify using the SBT launcher from other packages.
5+
# Note: This should be used by tools like giter8/conscript etc.
6+
7+
# TODO - Should we merge the main SBT script with this library?
8+
9+
if test -z "$HOME"; then
10+
declare -r script_dir="$(dirname $script_path)"
11+
else
12+
declare -r script_dir="$HOME/.sbt"
13+
fi
14+
15+
declare -a residual_args
16+
declare -a java_args
17+
declare -a scalac_args
18+
declare -a sbt_commands
19+
declare java_cmd=java
20+
21+
echoerr () {
22+
echo 1>&2 "$@"
23+
}
24+
vlog () {
25+
[[ $verbose || $debug ]] && echoerr "$@"
26+
}
27+
dlog () {
28+
[[ $debug ]] && echoerr "$@"
29+
}
30+
31+
acquire_sbt_jar () {
32+
SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
33+
URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
34+
URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
35+
JAR=sbt/sbt-launch-${SBT_VERSION}.jar
36+
37+
sbt_jar=$JAR
38+
39+
if [[ ! -f "$sbt_jar" ]]; then
40+
# Download sbt launch jar if it hasn't been downloaded yet
41+
if [ ! -f ${JAR} ]; then
42+
# Download
43+
printf "Attempting to fetch sbt\n"
44+
JAR_DL=${JAR}.part
45+
if hash curl 2>/dev/null; then
46+
(curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
47+
elif hash wget 2>/dev/null; then
48+
(wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
49+
else
50+
printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
51+
exit -1
52+
fi
53+
fi
54+
if [ ! -f ${JAR} ]; then
55+
# We failed to download
56+
printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
57+
exit -1
58+
fi
59+
printf "Launching sbt from ${JAR}\n"
60+
fi
61+
}
62+
63+
execRunner () {
64+
# print the arguments one to a line, quoting any containing spaces
65+
[[ $verbose || $debug ]] && echo "# Executing command line:" && {
66+
for arg; do
67+
if printf "%s\n" "$arg" | grep -q ' '; then
68+
printf "\"%s\"\n" "$arg"
69+
else
70+
printf "%s\n" "$arg"
71+
fi
72+
done
73+
echo ""
74+
}
75+
76+
exec "$@"
77+
}
78+
79+
addJava () {
80+
dlog "[addJava] arg = '$1'"
81+
java_args=( "${java_args[@]}" "$1" )
82+
}
83+
addSbt () {
84+
dlog "[addSbt] arg = '$1'"
85+
sbt_commands=( "${sbt_commands[@]}" "$1" )
86+
}
87+
addResidual () {
88+
dlog "[residual] arg = '$1'"
89+
residual_args=( "${residual_args[@]}" "$1" )
90+
}
91+
addDebugger () {
92+
addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
93+
}
94+
95+
# a ham-fisted attempt to move some memory settings in concert
96+
# so they need not be dicked around with individually.
97+
get_mem_opts () {
98+
local mem=${1:-2048}
99+
local perm=$(( $mem / 4 ))
100+
(( $perm > 256 )) || perm=256
101+
(( $perm < 1024 )) || perm=1024
102+
local codecache=$(( $perm / 2 ))
103+
104+
echo "-Xms${mem}m -Xmx${mem}m -XX:MaxPermSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m"
105+
}
106+
107+
require_arg () {
108+
local type="$1"
109+
local opt="$2"
110+
local arg="$3"
111+
if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
112+
die "$opt requires <$type> argument"
113+
fi
114+
}
115+
116+
is_function_defined() {
117+
declare -f "$1" > /dev/null
118+
}
119+
120+
process_args () {
121+
while [[ $# -gt 0 ]]; do
122+
case "$1" in
123+
-h|-help) usage; exit 1 ;;
124+
-v|-verbose) verbose=1 && shift ;;
125+
-d|-debug) debug=1 && shift ;;
126+
127+
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
128+
-mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;;
129+
-jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;
130+
-batch) exec </dev/null && shift ;;
131+
132+
-sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
133+
-sbt-version) require_arg version "$1" "$2" && sbt_version="$2" && shift 2 ;;
134+
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
135+
136+
-D*) addJava "$1" && shift ;;
137+
-J*) addJava "${1:2}" && shift ;;
138+
*) addResidual "$1" && shift ;;
139+
esac
140+
done
141+
142+
is_function_defined process_my_args && {
143+
myargs=("${residual_args[@]}")
144+
residual_args=()
145+
process_my_args "${myargs[@]}"
146+
}
147+
}
148+
149+
run() {
150+
# no jar? download it.
151+
[[ -f "$sbt_jar" ]] || acquire_sbt_jar "$sbt_version" || {
152+
# still no jar? uh-oh.
153+
echo "Download failed. Obtain the sbt-launch.jar manually and place it at $sbt_jar"
154+
exit 1
155+
}
156+
157+
# process the combined args, then reset "$@" to the residuals
158+
process_args "$@"
159+
set -- "${residual_args[@]}"
160+
argumentCount=$#
161+
162+
# run sbt
163+
execRunner "$java_cmd" \
164+
${SBT_OPTS:-$default_sbt_opts} \
165+
$(get_mem_opts $sbt_mem) \
166+
${java_opts} \
167+
${java_args[@]} \
168+
-jar "$sbt_jar" \
169+
"${sbt_commands[@]}" \
170+
"${residual_args[@]}"
171+
}
172+
173+
runAlternateBoot() {
174+
local bootpropsfile="$1"
175+
shift
176+
addJava "-Dsbt.boot.properties=$bootpropsfile"
177+
run $@
178+
}

0 commit comments

Comments
 (0)