upgrade hadoop to 2.8.0-palantir3

core/pom.xml

@@ -58,10 +58,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-launcher_${scala.binary.version}</artifactId>

dev/deps/spark-deps-hadoop-palantir

@@ -68,26 +68,25 @@ gson-2.2.4.jar
 guava-14.0.1.jar
 guice-3.0.jar
 guice-servlet-3.0.jar
-hadoop-annotations-2.8.0-palantir2.jar
-hadoop-auth-2.8.0-palantir2.jar
-hadoop-client-2.8.0-palantir2.jar
-hadoop-common-2.8.0-palantir2.jar
-hadoop-hdfs-2.8.0-palantir2.jar
-hadoop-hdfs-client-2.8.0-palantir2.jar
-hadoop-mapreduce-client-app-2.8.0-palantir2.jar
-hadoop-mapreduce-client-common-2.8.0-palantir2.jar
-hadoop-mapreduce-client-core-2.8.0-palantir2.jar
-hadoop-mapreduce-client-jobclient-2.8.0-palantir2.jar
-hadoop-mapreduce-client-shuffle-2.8.0-palantir2.jar
-hadoop-yarn-api-2.8.0-palantir2.jar
-hadoop-yarn-client-2.8.0-palantir2.jar
-hadoop-yarn-common-2.8.0-palantir2.jar
-hadoop-yarn-server-common-2.8.0-palantir2.jar
-hadoop-yarn-server-web-proxy-2.8.0-palantir2.jar
+hadoop-annotations-2.8.0-palantir3.jar
+hadoop-auth-2.8.0-palantir3.jar
+hadoop-client-2.8.0-palantir3.jar
+hadoop-common-2.8.0-palantir3.jar
+hadoop-hdfs-2.8.0-palantir3.jar
+hadoop-hdfs-client-2.8.0-palantir3.jar
+hadoop-mapreduce-client-app-2.8.0-palantir3.jar
+hadoop-mapreduce-client-common-2.8.0-palantir3.jar
+hadoop-mapreduce-client-core-2.8.0-palantir3.jar
+hadoop-mapreduce-client-jobclient-2.8.0-palantir3.jar
+hadoop-mapreduce-client-shuffle-2.8.0-palantir3.jar
+hadoop-yarn-api-2.8.0-palantir3.jar
+hadoop-yarn-client-2.8.0-palantir3.jar
+hadoop-yarn-common-2.8.0-palantir3.jar
+hadoop-yarn-server-common-2.8.0-palantir3.jar
+hadoop-yarn-server-web-proxy-2.8.0-palantir3.jar
 hk2-api-2.4.0-b34.jar
 hk2-locator-2.4.0-b34.jar
 hk2-utils-2.4.0-b34.jar
-hpack-0.11.0.jar
 htrace-core4-4.0.1-incubating.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
@@ -204,7 +203,6 @@ super-csv-2.2.0.jar
 univocity-parsers-2.2.1.jar
 validation-api-1.1.0.Final.jar
 xbean-asm5-shaded-4.4.jar
-xercesImpl-2.9.1.jar
 xmlenc-0.52.jar
 xz-1.0.jar
 zjsonpatch-0.3.0.jar

dev/publish.sh

@@ -27,5 +27,5 @@ make_dist() {
 }
 
 publish_artifacts
-make_dist hadoop-2.8.0-palantir2 "${PALANTIR_FLAGS[*]}" --clean
+make_dist hadoop-2.8.0-palantir3 "${PALANTIR_FLAGS[*]}" --clean
 make_dist without-hadoop "-Phadoop-provided -Pkubernetes -Pmesos -Pyarn -Psparkr" --clean

dists/hadoop-palantir/pom.xml

@@ -34,7 +34,7 @@
     <sbt.project.name>spark-dist-hadoop-palantir</sbt.project.name>
     <build.testJarPhase>none</build.testJarPhase>
     <build.copyDependenciesPhase>none</build.copyDependenciesPhase>
-    <hadoop.version>2.8.0-palantir2</hadoop.version>
+    <hadoop.version>2.8.0-palantir3</hadoop.version>
   </properties>
 
   <dependencyManagement>

pom.xml

@@ -918,79 +918,6 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <!-- TODO (srand) Remove this when https://issues.apache.org/jira/browse/HDFS-11431 is fixed -->
-      <dependency>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-hdfs</artifactId>
-        <version>${hadoop.version}</version>
-        <scope>${hadoop.deps.scope}</scope>
-        <exclusions>
-          <exclusion>
-            <groupId>asm</groupId>
-            <artifactId>asm</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-mapper-asl</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.ow2.asm</groupId>
-            <artifactId>asm</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.jboss.netty</groupId>
-            <artifactId>netty</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>io.netty</groupId>
-            <artifactId>netty</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-all</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commons-daemon</groupId>
-            <artifactId>commons-daemon</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commons-net</groupId>
-            <artifactId>commons-net</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-all</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>servlet-api-2.5</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>javax.servlet</groupId>
-            <artifactId>servlet-api</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>*</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey.jersey-test-framework</groupId>
-            <artifactId>*</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey.contribs</groupId>
-            <artifactId>*</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro</artifactId>
@@ -2656,7 +2583,7 @@
     <profile>
       <id>hadoop-palantir</id>
       <properties>
-        <hadoop.version>2.8.0-palantir2</hadoop.version>
+        <hadoop.version>2.8.0-palantir3</hadoop.version>
         <curator.version>2.7.1</curator.version>
       </properties>
     </profile>