Upgrade to Hadoop 3.x releases

changelog/@unreleased/pr-442.v2.yml

@@ -0,0 +1,5 @@
+type: break
+break:
+  description: Upgrade to Hadoop 3.x
+  links:
+  - https://github.com/palantir/hadoop-crypto/pull/442

hadoop-crypto/build.gradle

@@ -3,7 +3,10 @@ apply plugin: 'com.github.johnrengelman.shadow'
 
 dependencies {
     compile project(":crypto-core")
-    compile "org.apache.hadoop:hadoop-common"
+    compile "org.apache.hadoop:hadoop-client-api"
+    compile "org.slf4j:slf4j-api"
+
+    runtime "org.apache.hadoop:hadoop-client-runtime"
 
     testCompile "junit:junit"
     testCompile "org.assertj:assertj-core"
@@ -14,30 +17,20 @@ shadowJar {
     mergeServiceFiles()
 
     dependencies {
-        // only keep hadoop-aws, exclude all other hadoop-* jars.
-        exclude(dependency('org.apache.hadoop:hadoop-annotations'))
-        exclude(dependency('org.apache.hadoop:hadoop-auth'))
-        exclude(dependency('org.apache.hadoop:hadoop-client'))
-        exclude(dependency('org.apache.hadoop:hadoop-common'))
-        exclude(dependency('org.apache.hadoop:hadoop-hdfs'))
-        exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-app'))
-        exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-common'))
-        exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-core'))
-        exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-jobclient'))
-        exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-shuffle'))
-        exclude(dependency('org.apache.hadoop:hadoop-yarn-api'))
-        exclude(dependency('org.apache.hadoop:hadoop-yarn-client'))
-        exclude(dependency('org.apache.hadoop:hadoop-yarn-common'))
-        exclude(dependency('org.apache.hadoop:hadoop-yarn-server-common'))
-        exclude(dependency('org.apache.hadoop:hadoop-yarn-server-nodemanager'))
+        // The shadow jar is expected to be added to the Hadoop cli classpath
+        // which comes with these deps
+        exclude(dependency('org.apache.hadoop:hadoop-client-api'))
+        exclude(dependency('org.apache.hadoop:hadoop-client-runtime'))
     }
+}
+
+// Automatically shadow all included dependencies
+// https://imperceptiblethoughts.com/shadow/configuration/relocation/#automatically-relocating-dependencies
+import com.github.jengelman.gradle.plugins.shadow.tasks.ConfigureShadowRelocation
 
-    // Shade dependencies that can cause conflict with Hadoop
-    relocate('com.google.common', 'hadoop.crypto.shaded.com.google.common')
-    relocate('com.fasterxml.jackson', 'hadoop.crypto.shaded.com.fasterxml.jackson')
-    relocate('org.apache.commons.codec', 'hadoop.crypto.shaded.org.apache.commons.codec')
-    relocate('org.apache.commons.lang3', 'hadoop.crypto.shaded.org.apache.commons.lang3')
-    relocate('org.apache.commons.logging', 'hadoop.crypto.shaded.org.apache.commons.logging')
-    relocate('org.slf4j', 'hadoop.crypto.shaded.org.slf4j')
+task relocateShadowJar(type: ConfigureShadowRelocation) {
+    target = tasks.shadowJar
+    prefix = "hadoop.crypto.shaded" // Default value is "shadow"
 }
 
+tasks.shadowJar.dependsOn tasks.relocateShadowJar

hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/FileKeyStorageStrategy.java

@@ -18,6 +18,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
+import com.google.common.io.ByteStreams;
 import com.palantir.crypto2.keys.KeyMaterial;
 import com.palantir.crypto2.keys.KeyStorageStrategy;
 import com.palantir.crypto2.keys.serialization.KeyMaterials;
@@ -29,7 +30,6 @@
 import java.security.PublicKey;
 import java.util.Optional;
 import javax.crypto.SecretKey;
-import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -61,7 +61,7 @@ public FileKeyStorageStrategy(FileSystem fs, PublicKey publicKey) {
     public void put(String fileKey, KeyMaterial keyMaterial) {
         try (OutputStream stream = fs.create(getKeyPath(fileKey))) {
             byte[] wrappedKey = KeyMaterials.wrap(keyMaterial, publicKey);
-            IOUtils.write(wrappedKey, stream);
+            stream.write(wrappedKey);
         } catch (IOException e) {
             throw Throwables.propagate(e);
         }
@@ -71,7 +71,7 @@ public void put(String fileKey, KeyMaterial keyMaterial) {
     public KeyMaterial get(String fileKey) {
         Preconditions.checkArgument(privateKey.isPresent(), "Private key is absent but required to get key material");
         try (InputStream stream = fs.open(getKeyPath(fileKey))) {
-            byte[] wrappedKey = IOUtils.toByteArray(stream);
+            byte[] wrappedKey = ByteStreams.toByteArray(stream);
             return KeyMaterials.unwrap(wrappedKey, privateKey.get());
         } catch (IOException e) {
             throw Throwables.propagate(e);

hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystem.java

@@ -23,11 +23,11 @@
 import com.palantir.crypto2.keys.KeyStorageStrategy;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.KeyPair;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.function.Function;
-import javax.ws.rs.core.UriBuilder;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -163,17 +163,23 @@ private static Function<Path, Path> setSchemeFunc(final String scheme) {
 
     private static Function<URI, URI> setUriSchemeFunc(final String scheme) {
         return uri -> {
-            UriBuilder builder = UriBuilder.fromUri(uri);
-            if (uri.getScheme() != null) {
-                builder.scheme(scheme);
+            try {
+                return new URI(
+                        scheme,
+                        uri.getUserInfo(),
+                        uri.getHost(),
+                        uri.getPort(),
+                        uri.getPath(),
+                        uri.getQuery(),
+                        uri.getFragment());
+            } catch (URISyntaxException e) {
+                throw new RuntimeException(e);
             }
-            return builder.build();
         };
     }
 
     @Override
     public String getScheme() {
         return SCHEME;
     }
-
 }

hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/EncryptedFileSystemTest.java

@@ -28,21 +28,21 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
+import com.google.common.io.ByteStreams;
 import com.palantir.crypto2.cipher.AesCtrCipher;
 import com.palantir.crypto2.keys.KeyMaterial;
 import com.palantir.crypto2.keys.KeyStorageStrategy;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Random;
-import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -118,32 +118,30 @@ public void testDelegateStreamIsClosed() throws IOException {
     @Test
     public void testEncryptDecrypt_success() throws IllegalArgumentException, IOException {
         byte[] data = new byte[MB];
-        byte[] readData = new byte[MB];
         random.nextBytes(data);
 
         OutputStream os = efs.create(path);
-        IOUtils.write(data, os);
+        os.write(data);
         os.close();
 
         // Read using EncryptedFileSystem yields input data
         InputStream is = efs.open(path);
-        IOUtils.readFully(is, readData);
+        byte[] readData = ByteStreams.toByteArray(is);
         is.close();
 
         assertThat(data).isEqualTo(readData);
 
         // Read using delegate FileSystem does not yield input data
         is = delegateFs.open(path);
-        IOUtils.readFully(is, readData);
+        byte[] rawData = ByteStreams.toByteArray(is);
         is.close();
 
-        assertThat(data).isNotEqualTo(readData);
+        assertThat(data).isNotEqualTo(rawData);
     }
 
     @Test
     public void testEncryptDecrypt_secondaryCreateMethod() throws IOException {
         byte[] data = new byte[MB];
-        byte[] readData = new byte[MB];
         random.nextBytes(data);
 
         OutputStream os = efs.create(
@@ -155,38 +153,37 @@ public void testEncryptDecrypt_secondaryCreateMethod() throws IOException {
                 64 * 1024 * 1024,
                 null,
                 null);
-        IOUtils.write(data, os);
+        os.write(data);
         os.close();
 
         // Read using EncryptedFileSystem yields input data
         InputStream is = efs.open(path);
-        IOUtils.readFully(is, readData);
+        byte[] readData = ByteStreams.toByteArray(is);
         is.close();
 
         assertThat(data).isEqualTo(readData);
 
         // Read using delegate FileSystem does not yield input data
         is = delegateFs.open(path);
-        IOUtils.readFully(is, readData);
+        byte[] rawData = ByteStreams.toByteArray(is);
         is.close();
 
-        assertThat(data).isNotEqualTo(readData);
+        assertThat(data).isNotEqualTo(rawData);
     }
 
     @Test
     public void testEncryptDecrypt_decryptSeek() throws IllegalArgumentException, IOException {
         byte[] data = new byte[MB];
         int seekPos = MB / 2;
-        byte[] readData = new byte[MB - seekPos];
         random.nextBytes(data);
 
         OutputStream os = efs.create(path);
-        IOUtils.write(data, os);
+        os.write(data);
         os.close();
 
         FSDataInputStream is = efs.open(path);
         is.seek(seekPos);
-        IOUtils.readFully(is, readData);
+        byte[] readData = ByteStreams.toByteArray(is);
 
         byte[] actualReadData = Arrays.copyOfRange(data, seekPos, MB);
         assertThat(actualReadData).isEqualTo(readData);
@@ -390,13 +387,12 @@ public void testAppend() {
     public void testCopyFromLocalFile() throws IOException {
         File file = folder.newFile();
         byte[] data = "data".getBytes(StandardCharsets.UTF_8);
-        byte[] readBytes = new byte[data.length];
 
-        IOUtils.write(data, new FileOutputStream(file));
+        Files.write(file.toPath(), data);
         efs.copyFromLocalFile(new Path(file.getAbsolutePath()), path);
 
         FSDataInputStream input = efs.open(path);
-        IOUtils.readFully(input, readBytes);
+        byte[] readBytes = ByteStreams.toByteArray(input);
         assertThat(readBytes).isEqualTo(data);
     }
 

hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystemTest.java

@@ -21,7 +21,6 @@
 
 import com.google.common.io.ByteStreams;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -31,7 +30,6 @@
 import java.security.KeyPair;
 import java.util.Base64;
 import java.util.UUID;
-import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -83,22 +81,21 @@ public void testGetUri_schemeIsCorrect() {
 
     @Test
     public void testReadWrite() throws IOException {
-        byte[] readData = new byte[DATA.length()];
-
         // Write encrypted data
         OutputStream os = efs.create(path);
-        IOUtils.write(DATA, os);
+
+        os.write(DATA_BYTES);
         os.close();
 
         // Read encrypted data
         InputStream dis = efs.open(path);
-        IOUtils.readFully(dis, readData);
-        assertThat(readData).containsExactly(DATA_BYTES);
+        byte[] encryptedData = ByteStreams.toByteArray(dis);
+        assertThat(encryptedData).containsExactly(DATA_BYTES);
 
         // Raw data is not the same
         dis = rawFs.open(path);
-        IOUtils.readFully(dis, readData);
-        assertThat(readData).isNotEqualTo(DATA_BYTES);
+        byte[] rawData = ByteStreams.toByteArray(dis);
+        assertThat(rawData).isNotEqualTo(DATA_BYTES);
 
         // KeyMaterial file exists
         assertThat(rawFs.exists(keyMaterialPath(path))).isTrue();
@@ -202,34 +199,34 @@ private Path writeData(File rootFolder) throws IOException {
         // Write encrypted data
         java.nio.file.Path filePath = Files.createTempFile(rootFolder.toPath(), "prefix", "suffix");
         Path newPath = new Path(filePath.toAbsolutePath().toString());
-        OutputStream os = efs.create(newPath);
-        IOUtils.write(DATA, os);
-        os.close();
+
+        try (OutputStream os = efs.create(newPath)) {
+            os.write(DATA_BYTES);
+        }
 
         return newPath;
     }
 
     @Test
-    public void testMakeQualified() throws IOException {
+    public void testMakeQualified() {
         assertThat(efs.makeQualified(pathWithScheme)).isEqualTo(pathWithScheme);
     }
 
     @Test
     public void testOnlyPublicKey() throws IOException {
         byte[] dataBytes = DATA.getBytes(StandardCharsets.UTF_8);
-        byte[] readData = new byte[DATA.length()];
 
         conf.unset(StandaloneEncryptedFileSystem.PRIVATE_KEY_CONF);
         FileSystem efsPublic = FileSystem.newInstance(EFS_URI, conf);
 
         // Write encrypted data
         OutputStream os = efsPublic.create(path);
-        IOUtils.write(DATA, os);
+        os.write(DATA_BYTES);
         os.close();
 
         // Raw data is not the same
         InputStream dis = rawFs.open(path);
-        IOUtils.readFully(dis, readData);
+        byte[] readData = ByteStreams.toByteArray(dis);
         assertThat(readData).isNotEqualTo(dataBytes);
 
         // KeyMaterial file exists
@@ -242,14 +239,14 @@ public void testOnlyPublicKey() throws IOException {
     }
 
     @Test
-    public void testNoPublicKey() throws IOException {
+    public void testNoPublicKey() {
         assertThatExceptionOfType(NullPointerException.class)
                 .isThrownBy(() -> FileSystem.newInstance(EFS_URI, getBaseConf()))
                 .withMessage("Public Key must be configured for key %s", StandaloneEncryptedFileSystem.PUBLIC_KEY_CONF);
     }
 
     @Test
-    public void testBackingFsInvalid() throws IOException {
+    public void testBackingFsInvalid() {
         conf = getBaseConf();
         conf.set("fs.nope.impl", StandaloneEncryptedFileSystem.class.getCanonicalName());
 
@@ -262,12 +259,12 @@ public void testBackingFsInvalid() throws IOException {
     public void testFileNameCollidesWithKeyMaterial() throws IOException {
         // Write encrypted data
         OutputStream os = efs.create(path);
-        IOUtils.write(DATA, os);
+        os.write(DATA_BYTES);
         os.close();
 
         // Write encrypted data
         os = efs.create(keyMaterialPath(path));
-        IOUtils.write(DATA, os);
+        os.write(DATA_BYTES);
         os.close();
 
         // NOTE(jellis): IllegalArgumentException is the most likely exception, however if the first byte of the
@@ -294,13 +291,12 @@ public void testListStatus_keyMaterialFilesFiltered() throws IOException {
     @Test // https://github.com/palantir/hadoop-crypto/issues/27
     public void testCopyFromLocalFile() throws IOException {
         File file = folder.newFile();
-        byte[] readBytes = new byte[DATA_BYTES.length];
 
-        IOUtils.write(DATA_BYTES, new FileOutputStream(file));
+        Files.write(file.toPath(), DATA_BYTES);
         efs.copyFromLocalFile(new Path(file.getAbsolutePath()), path);
 
         FSDataInputStream input = efs.open(path);
-        IOUtils.readFully(input, readBytes);
+        byte[] readBytes = ByteStreams.toByteArray(input);
         assertThat(readBytes).containsExactly(DATA_BYTES);
     }