Skip to content

Commit 2a0af1d

Browse files
YY-OnCallYY-OnCall
committed
resolve merge conflict
2 parents bd18c00 + 1a45d2b commit 2a0af1d

File tree

1,618 files changed

+59211
-25318
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,618 files changed

+59211
-25318
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ dependency-reduced-pom.xml
4242
derby.log
4343
dev/create-release/*final
4444
dev/create-release/*txt
45+
dev/pr-deps/
4546
dist/
4647
docs/_site
4748
docs/api

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ notifications:
4444
# 5. Run maven install before running lint-java.
4545
install:
4646
- export MAVEN_SKIP_RC=1
47-
- build/mvn -T 4 -q -DskipTests -Pmesos -Pyarn -Phadoop-2.3 -Pkinesis-asl -Phive -Phive-thriftserver install
47+
- build/mvn -T 4 -q -DskipTests -Pmesos -Pyarn -Pkinesis-asl -Phive -Phive-thriftserver install
4848

4949
# 6. Run lint-java.
5050
script:

‎R/CRAN_RELEASE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ To release SparkR as a package to CRAN, we would use the `devtools` package. Ple
77

88
First, check that the `Version:` field in the `pkg/DESCRIPTION` file is updated. Also, check for stale files not under source control.
99

10-
Note that while `check-cran.sh` is running `R CMD check`, it is doing so with `--no-manual --no-vignettes`, which skips a few vignettes or PDF checks - therefore it will be preferred to run `R CMD check` on the source package built manually before uploading a release.
10+
Note that while `run-tests.sh` runs `check-cran.sh` (which runs `R CMD check`), it is doing so with `--no-manual --no-vignettes`, which skips a few vignettes or PDF checks - therefore it will be preferred to run `R CMD check` on the source package built manually before uploading a release. Also note that for CRAN checks for pdf vignettes to success, `qpdf` tool must be there (to install it, eg. `yum -q -y install qpdf`).
1111

1212
To upload a release, we would need to update the `cran-comments.md`. This should generally contain the results from running the `check-cran.sh` script along with comments on status of all `WARNING` (should not be any) or `NOTE`. As a part of `check-cran.sh` and the release process, the vignettes is build - make sure `SPARK_HOME` is set and Spark jars are accessible.
1313

‎R/check-cran.sh‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,14 @@
2020
set -o pipefail
2121
set -e
2222

23-
FWDIR="$(cd `dirname $0`; pwd)"
23+
FWDIR="$(cd `dirname "${BASH_SOURCE[0]}"`; pwd)"
2424
pushd $FWDIR > /dev/null
2525

26-
if [ ! -z "$R_HOME" ]
27-
then
28-
R_SCRIPT_PATH="$R_HOME/bin"
29-
else
30-
# if system wide R_HOME is not found, then exit
31-
if [ ! `command -v R` ]; then
32-
echo "Cannot find 'R_HOME'. Please specify 'R_HOME' or make sure R is properly installed."
33-
exit 1
34-
fi
35-
R_SCRIPT_PATH="$(dirname $(which R))"
36-
fi
37-
echo "USING R_HOME = $R_HOME"
26+
. $FWDIR/find-r.sh
3827

28+
# Install the package (this is required for code in vignettes to run when building it later)
3929
# Build the latest docs, but not vignettes, which is built with the package next
40-
$FWDIR/create-docs.sh
30+
. $FWDIR/install-dev.sh
4131

4232
# Build source package with vignettes
4333
SPARK_HOME="$(cd "${FWDIR}"/..; pwd)"
@@ -82,4 +72,5 @@ else
8272
# This will run tests and/or build vignettes, and require SPARK_HOME
8373
SPARK_HOME="${SPARK_HOME}" "$R_SCRIPT_PATH/"R CMD check $CRAN_CHECK_OPTIONS SparkR_"$VERSION".tar.gz
8474
fi
75+
8576
popd > /dev/null

‎R/create-docs.sh‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,26 +29,27 @@ set -o pipefail
2929
set -e
3030

3131
# Figure out where the script is
32-
export FWDIR="$(cd "`dirname "$0"`"; pwd)"
33-
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
32+
export FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
33+
export SPARK_HOME="$(cd "`dirname "${BASH_SOURCE[0]}"`"/..; pwd)"
3434

3535
# Required for setting SPARK_SCALA_VERSION
3636
. "${SPARK_HOME}"/bin/load-spark-env.sh
3737

3838
echo "Using Scala $SPARK_SCALA_VERSION"
3939

40-
pushd $FWDIR
40+
pushd $FWDIR > /dev/null
41+
. $FWDIR/find-r.sh
4142

4243
# Install the package (this will also generate the Rd files)
43-
./install-dev.sh
44+
. $FWDIR/install-dev.sh
4445

4546
# Now create HTML files
4647

4748
# knit_rd puts html in current working directory
4849
mkdir -p pkg/html
4950
pushd pkg/html
5051

51-
Rscript -e 'libDir <- "../../lib"; library(SparkR, lib.loc=libDir); library(knitr); knit_rd("SparkR", links = tools::findHTMLlinks(paste(libDir, "SparkR", sep="/")))'
52+
"$R_SCRIPT_PATH/"Rscript -e 'libDir <- "../../lib"; library(SparkR, lib.loc=libDir); library(knitr); knit_rd("SparkR", links = tools::findHTMLlinks(paste(libDir, "SparkR", sep="/")))'
5253

5354
popd
5455

‎R/create-rd.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# This scripts packages the SparkR source files (R and C files) and
21+
# creates a package that can be loaded in R. The package is by default installed to
22+
# $FWDIR/lib and the package can be loaded by using the following command in R:
23+
#
24+
# library(SparkR, lib.loc="$FWDIR/lib")
25+
#
26+
# NOTE(shivaram): Right now we use $SPARK_HOME/R/lib to be the installation directory
27+
# to load the SparkR package on the worker nodes.
28+
29+
set -o pipefail
30+
set -e
31+
32+
FWDIR="$(cd `dirname "${BASH_SOURCE[0]}"`; pwd)"
33+
pushd $FWDIR > /dev/null
34+
. $FWDIR/find-r.sh
35+
36+
# Generate Rd files if devtools is installed
37+
"$R_SCRIPT_PATH/"Rscript -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); devtools::document(pkg="./pkg", roclets=c("rd")) }'

external/java8-tests/src/test/resources/log4j.properties renamed to R/find-r.sh

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/bin/bash
2+
13
#
24
# Licensed to the Apache Software Foundation (ASF) under one or more
35
# contributor license agreements. See the NOTICE file distributed with
@@ -15,13 +17,18 @@
1517
# limitations under the License.
1618
#
1719

18-
# Set everything to be logged to the file target/unit-tests.log
19-
log4j.rootCategory=INFO, file
20-
log4j.appender.file=org.apache.log4j.FileAppender
21-
log4j.appender.file.append=true
22-
log4j.appender.file.file=target/unit-tests.log
23-
log4j.appender.file.layout=org.apache.log4j.PatternLayout
24-
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
25-
26-
# Ignore messages below warning level from Jetty, because it's a bit verbose
27-
log4j.logger.org.spark_project.jetty=WARN
20+
if [ -z "$R_SCRIPT_PATH" ]
21+
then
22+
if [ ! -z "$R_HOME" ]
23+
then
24+
R_SCRIPT_PATH="$R_HOME/bin"
25+
else
26+
# if system wide R_HOME is not found, then exit
27+
if [ ! `command -v R` ]; then
28+
echo "Cannot find 'R_HOME'. Please specify 'R_HOME' or make sure R is properly installed."
29+
exit 1
30+
fi
31+
R_SCRIPT_PATH="$(dirname $(which R))"
32+
fi
33+
echo "Using R_SCRIPT_PATH = ${R_SCRIPT_PATH}"
34+
fi

‎R/install-dev.sh‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,15 @@
2929
set -o pipefail
3030
set -e
3131

32-
FWDIR="$(cd `dirname $0`; pwd)"
32+
FWDIR="$(cd `dirname "${BASH_SOURCE[0]}"`; pwd)"
3333
LIB_DIR="$FWDIR/lib"
3434

3535
mkdir -p $LIB_DIR
3636

3737
pushd $FWDIR > /dev/null
38-
if [ ! -z "$R_HOME" ]
39-
then
40-
R_SCRIPT_PATH="$R_HOME/bin"
41-
else
42-
# if system wide R_HOME is not found, then exit
43-
if [ ! `command -v R` ]; then
44-
echo "Cannot find 'R_HOME'. Please specify 'R_HOME' or make sure R is properly installed."
45-
exit 1
46-
fi
47-
R_SCRIPT_PATH="$(dirname $(which R))"
48-
fi
49-
echo "USING R_HOME = $R_HOME"
50-
51-
# Generate Rd files if devtools is installed
52-
"$R_SCRIPT_PATH/"Rscript -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); devtools::document(pkg="./pkg", roclets=c("rd")) }'
38+
. $FWDIR/find-r.sh
39+
40+
. $FWDIR/create-rd.sh
5341

5442
# Install SparkR to $LIB_DIR
5543
"$R_SCRIPT_PATH/"R CMD INSTALL --library=$LIB_DIR $FWDIR/pkg/

‎R/install-source-package.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# This scripts packages the SparkR source files (R and C files) and
21+
# creates a package that can be loaded in R. The package is by default installed to
22+
# $FWDIR/lib and the package can be loaded by using the following command in R:
23+
#
24+
# library(SparkR, lib.loc="$FWDIR/lib")
25+
#
26+
# NOTE(shivaram): Right now we use $SPARK_HOME/R/lib to be the installation directory
27+
# to load the SparkR package on the worker nodes.
28+
29+
set -o pipefail
30+
set -e
31+
32+
FWDIR="$(cd `dirname "${BASH_SOURCE[0]}"`; pwd)"
33+
pushd $FWDIR > /dev/null
34+
. $FWDIR/find-r.sh
35+
36+
if [ -z "$VERSION" ]; then
37+
VERSION=`grep Version $FWDIR/pkg/DESCRIPTION | awk '{print $NF}'`
38+
fi
39+
40+
if [ ! -f "$FWDIR"/SparkR_"$VERSION".tar.gz ]; then
41+
echo -e "R source package file $FWDIR/SparkR_$VERSION.tar.gz is not found."
42+
echo -e "Please build R source package with check-cran.sh"
43+
exit -1;
44+
fi
45+
46+
echo "Removing lib path and installing from source package"
47+
LIB_DIR="$FWDIR/lib"
48+
rm -rf $LIB_DIR
49+
mkdir -p $LIB_DIR
50+
"$R_SCRIPT_PATH/"R CMD INSTALL SparkR_"$VERSION".tar.gz --library=$LIB_DIR
51+
52+
# Zip the SparkR package so that it can be distributed to worker nodes on YARN
53+
pushd $LIB_DIR > /dev/null
54+
jar cfM "$LIB_DIR/sparkr.zip" SparkR
55+
popd > /dev/null
56+
57+
popd

‎R/pkg/.Rbuildignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
^.*\.Rproj$
22
^\.Rproj\.user$
33
^\.lintr$
4+
^cran-comments\.md$
5+
^NEWS\.md$
6+
^README\.Rmd$
47
^src-native$
58
^html$

0 commit comments

Comments
 (0)