diff --git a/js/downloads.js b/js/downloads.js
index 80dbdcbb834..d817dbd6e31 100644
--- a/js/downloads.js
+++ b/js/downloads.js
@@ -3,8 +3,13 @@
releases = {};
-function addRelease(version, releaseDate, packages, stable) {
- releases[version] = {released: releaseDate, packages: packages, stable: stable};
+function addRelease(version, releaseDate, packages, stable, mirrored) {
+ releases[version] = {
+ released: releaseDate,
+ packages: packages,
+ stable: stable,
+ mirrored: mirrored
+ };
}
var sources = {pretty: "Source Code", tag: "sources"};
@@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source
// 2.2.0+
var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources];
-addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true);
-addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true);
-addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true);
-addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true);
-addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true);
-addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true);
-addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true);
-addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true);
-addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true);
-addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true);
+addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true);
+addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true);
+addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true);
+addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false);
+addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true);
+addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false);
+addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false);
+addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true);
+addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false);
+addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false);
//addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false);
-addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true);
-addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true);
-addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true);
-addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true);
-//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true);
-//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true);
-//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true);
-//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true);
-//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true);
+addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true);
+addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false);
+addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false);
+addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false);
function append(el, contents) {
el.innerHTML += contents;
@@ -128,10 +128,10 @@ function onVersionSelect() {
append(verifyLink, link);
// Populate releases
- updateDownloadLink();
+ updateDownloadLink(releases[version].mirrored);
}
-function updateDownloadLink() {
+function updateDownloadLink(isMirrored) {
var versionSelect = document.getElementById("sparkVersionSelect");
var packageSelect = document.getElementById("sparkPackageSelect");
var downloadLink = document.getElementById("spanDownloadLink");
@@ -147,12 +147,10 @@ function updateDownloadLink() {
.replace(/-bin-sources/, ""); // special case for source packages
var link = "";
- if (version < "1.6.3" ||
- (version >= "2.0.0" && version <= "2.0.1") ||
- (version >= "2.1.0" && version <= "2.1.1")) {
- link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
- } else {
+ if (isMirrored) {
link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact";
+ } else {
+ link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
}
link = link
.replace(/\$ver/, version)
diff --git a/release-process.md b/release-process.md
index e756cc0b56a..b2ee62ba98d 100644
--- a/release-process.md
+++ b/release-process.md
@@ -177,26 +177,38 @@ Publishing to CRAN is done using PMC.
-
Remove Old Releases from Development Repository and Mirror Network
+ Remove RC artifacts from repositories
-Spark always keeps two releases in the mirror network: the most recent release on the current and
-previous branches. To delete older versions simply use svn rm. The `downloads.js` file in the
-website `js/` directory must also be updated to reflect the changes. For instance, the two
-releases should be 1.1.1 and 1.0.2, but not 1.1.1 and 1.1.0.
-
-```
-$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
-```
-
-You should also delete the RC directories from the staging repository. For example:
+After the vote passes and you moved the approved RC to the release repository, you should delete
+the RC directories from the staging repository. For example:
```
svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
-m"Removing RC artifacts."
+```
+
+Make sure to also remove the unpublished staging repositories from the
+Apache Nexus Repository Manager.
+
+Remove Old Releases from Mirror Network
+
+Spark always keeps the latest maintance released of each branch in the mirror network.
+To delete older versions simply use svn rm:
+
+```
+$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
```
+You will also need to update `js/download.js` to indicate the release is not mirrored
+anymore, so that the correct links are generated on the site.
+
+Also take a moment to check `HiveExternalCatalogVersionsSuite.scala` starting with branch-2.2
+and see if it needs to be adjusted, since that test relies on mirrored downloads of previous
+releases.
+
+
Update the Spark Apache Repository
Check out the tagged commit for the release candidate that passed and apply the correct version tag.
diff --git a/site/doap.rdf b/site/doap.rdf
index 1568c3fdea8..25bb8fc4be5 100644
--- a/site/doap.rdf
+++ b/site/doap.rdf
@@ -28,13 +28,16 @@
Apache Spark is a fast and general engine for large-scale data processing.
- Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala and Python as well as a rich set of libraries including stream processing, machine learning, and graph analytics.
+ Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala, Python and R, as well as a rich set of libraries including stream processing, machine learning, and graph analytics.
Java
Scala
Python
+ R
+ SQL
+ JDBC
diff --git a/site/js/downloads.js b/site/js/downloads.js
index 80dbdcbb834..d817dbd6e31 100644
--- a/site/js/downloads.js
+++ b/site/js/downloads.js
@@ -3,8 +3,13 @@
releases = {};
-function addRelease(version, releaseDate, packages, stable) {
- releases[version] = {released: releaseDate, packages: packages, stable: stable};
+function addRelease(version, releaseDate, packages, stable, mirrored) {
+ releases[version] = {
+ released: releaseDate,
+ packages: packages,
+ stable: stable,
+ mirrored: mirrored
+ };
}
var sources = {pretty: "Source Code", tag: "sources"};
@@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source
// 2.2.0+
var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources];
-addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true);
-addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true);
-addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true);
-addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true);
-addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true);
-addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true);
-addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true);
-addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true);
-addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true);
-addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true);
+addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true);
+addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true);
+addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true);
+addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false);
+addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true);
+addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false);
+addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false);
+addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true);
+addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false);
+addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false);
//addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false);
-addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true);
-addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true);
-addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true);
-addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true);
-//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true);
-//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true);
-//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true);
-//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true);
-//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true);
+addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true);
+addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false);
+addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false);
+addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false);
function append(el, contents) {
el.innerHTML += contents;
@@ -128,10 +128,10 @@ function onVersionSelect() {
append(verifyLink, link);
// Populate releases
- updateDownloadLink();
+ updateDownloadLink(releases[version].mirrored);
}
-function updateDownloadLink() {
+function updateDownloadLink(isMirrored) {
var versionSelect = document.getElementById("sparkVersionSelect");
var packageSelect = document.getElementById("sparkPackageSelect");
var downloadLink = document.getElementById("spanDownloadLink");
@@ -147,12 +147,10 @@ function updateDownloadLink() {
.replace(/-bin-sources/, ""); // special case for source packages
var link = "";
- if (version < "1.6.3" ||
- (version >= "2.0.0" && version <= "2.0.1") ||
- (version >= "2.1.0" && version <= "2.1.1")) {
- link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
- } else {
+ if (isMirrored) {
link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact";
+ } else {
+ link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
}
link = link
.replace(/\$ver/, version)
diff --git a/site/release-process.html b/site/release-process.html
index a3b311d6981..24157989106 100644
--- a/site/release-process.html
+++ b/site/release-process.html
@@ -376,24 +376,34 @@ Publish to CRAN
Publishing to CRAN is done using this form.
Since it requires further manual steps, please also contact the PMC.
-Remove Old Releases from Development Repository and Mirror Network
+ Remove RC artifacts from repositories
-Spark always keeps two releases in the mirror network: the most recent release on the current and
-previous branches. To delete older versions simply use svn rm. The downloads.js file in the
-website js/ directory must also be updated to reflect the changes. For instance, the two
-releases should be 1.1.1 and 1.0.2, but not 1.1.1 and 1.1.0.
-
-$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
-
-
-You should also delete the RC directories from the staging repository. For example:
+After the vote passes and you moved the approved RC to the release repository, you should delete
+the RC directories from the staging repository. For example:
svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
-m"Removing RC artifacts."
+
+
+Make sure to also remove the unpublished staging repositories from the
+Apache Nexus Repository Manager.
+Remove Old Releases from Mirror Network
+
+Spark always keeps the latest maintance released of each branch in the mirror network.
+To delete older versions simply use svn rm:
+
+$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
+You will also need to update js/download.js to indicate the release is not mirrored
+anymore, so that the correct links are generated on the site.
+
+Also take a moment to check HiveExternalCatalogVersionsSuite.scala starting with branch-2.2
+and see if it needs to be adjusted, since that test relies on mirrored downloads of previous
+releases.
+
Update the Spark Apache Repository
Check out the tagged commit for the release candidate that passed and apply the correct version tag.