diff --git a/changelog/@unreleased/pr-442.v2.yml b/changelog/@unreleased/pr-442.v2.yml new file mode 100644 index 000000000..a50cde77e --- /dev/null +++ b/changelog/@unreleased/pr-442.v2.yml @@ -0,0 +1,5 @@ +type: break +break: + description: Upgrade to Hadoop 3.x + links: + - https://github.com/palantir/hadoop-crypto/pull/442 diff --git a/hadoop-crypto/build.gradle b/hadoop-crypto/build.gradle index 846abef62..96db31035 100644 --- a/hadoop-crypto/build.gradle +++ b/hadoop-crypto/build.gradle @@ -3,7 +3,10 @@ apply plugin: 'com.github.johnrengelman.shadow' dependencies { compile project(":crypto-core") - compile "org.apache.hadoop:hadoop-common" + compile "org.apache.hadoop:hadoop-client-api" + compile "org.slf4j:slf4j-api" + + runtime "org.apache.hadoop:hadoop-client-runtime" testCompile "junit:junit" testCompile "org.assertj:assertj-core" @@ -14,30 +17,20 @@ shadowJar { mergeServiceFiles() dependencies { - // only keep hadoop-aws, exclude all other hadoop-* jars. - exclude(dependency('org.apache.hadoop:hadoop-annotations')) - exclude(dependency('org.apache.hadoop:hadoop-auth')) - exclude(dependency('org.apache.hadoop:hadoop-client')) - exclude(dependency('org.apache.hadoop:hadoop-common')) - exclude(dependency('org.apache.hadoop:hadoop-hdfs')) - exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-app')) - exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-common')) - exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-core')) - exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-jobclient')) - exclude(dependency('org.apache.hadoop:hadoop-mapreduce-client-shuffle')) - exclude(dependency('org.apache.hadoop:hadoop-yarn-api')) - exclude(dependency('org.apache.hadoop:hadoop-yarn-client')) - exclude(dependency('org.apache.hadoop:hadoop-yarn-common')) - exclude(dependency('org.apache.hadoop:hadoop-yarn-server-common')) - exclude(dependency('org.apache.hadoop:hadoop-yarn-server-nodemanager')) + // The shadow jar is expected to be added to the Hadoop cli classpath + // which comes with these deps + exclude(dependency('org.apache.hadoop:hadoop-client-api')) + exclude(dependency('org.apache.hadoop:hadoop-client-runtime')) } +} + +// Automatically shadow all included dependencies +// https://imperceptiblethoughts.com/shadow/configuration/relocation/#automatically-relocating-dependencies +import com.github.jengelman.gradle.plugins.shadow.tasks.ConfigureShadowRelocation - // Shade dependencies that can cause conflict with Hadoop - relocate('com.google.common', 'hadoop.crypto.shaded.com.google.common') - relocate('com.fasterxml.jackson', 'hadoop.crypto.shaded.com.fasterxml.jackson') - relocate('org.apache.commons.codec', 'hadoop.crypto.shaded.org.apache.commons.codec') - relocate('org.apache.commons.lang3', 'hadoop.crypto.shaded.org.apache.commons.lang3') - relocate('org.apache.commons.logging', 'hadoop.crypto.shaded.org.apache.commons.logging') - relocate('org.slf4j', 'hadoop.crypto.shaded.org.slf4j') +task relocateShadowJar(type: ConfigureShadowRelocation) { + target = tasks.shadowJar + prefix = "hadoop.crypto.shaded" // Default value is "shadow" } +tasks.shadowJar.dependsOn tasks.relocateShadowJar diff --git a/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/FileKeyStorageStrategy.java b/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/FileKeyStorageStrategy.java index 9bcd6e054..3aef7a710 100644 --- a/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/FileKeyStorageStrategy.java +++ b/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/FileKeyStorageStrategy.java @@ -18,6 +18,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.io.ByteStreams; import com.palantir.crypto2.keys.KeyMaterial; import com.palantir.crypto2.keys.KeyStorageStrategy; import com.palantir.crypto2.keys.serialization.KeyMaterials; @@ -29,7 +30,6 @@ import java.security.PublicKey; import java.util.Optional; import javax.crypto.SecretKey; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -61,7 +61,7 @@ public FileKeyStorageStrategy(FileSystem fs, PublicKey publicKey) { public void put(String fileKey, KeyMaterial keyMaterial) { try (OutputStream stream = fs.create(getKeyPath(fileKey))) { byte[] wrappedKey = KeyMaterials.wrap(keyMaterial, publicKey); - IOUtils.write(wrappedKey, stream); + stream.write(wrappedKey); } catch (IOException e) { throw Throwables.propagate(e); } @@ -71,7 +71,7 @@ public void put(String fileKey, KeyMaterial keyMaterial) { public KeyMaterial get(String fileKey) { Preconditions.checkArgument(privateKey.isPresent(), "Private key is absent but required to get key material"); try (InputStream stream = fs.open(getKeyPath(fileKey))) { - byte[] wrappedKey = IOUtils.toByteArray(stream); + byte[] wrappedKey = ByteStreams.toByteArray(stream); return KeyMaterials.unwrap(wrappedKey, privateKey.get()); } catch (IOException e) { throw Throwables.propagate(e); diff --git a/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystem.java b/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystem.java index 8e4f9754c..a53b719ef 100644 --- a/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystem.java +++ b/hadoop-crypto/src/main/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystem.java @@ -23,11 +23,11 @@ import com.palantir.crypto2.keys.KeyStorageStrategy; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.security.KeyPair; import java.util.Arrays; import java.util.Collection; import java.util.function.Function; -import javax.ws.rs.core.UriBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -163,11 +163,18 @@ private static Function setSchemeFunc(final String scheme) { private static Function setUriSchemeFunc(final String scheme) { return uri -> { - UriBuilder builder = UriBuilder.fromUri(uri); - if (uri.getScheme() != null) { - builder.scheme(scheme); + try { + return new URI( + scheme, + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + uri.getPath(), + uri.getQuery(), + uri.getFragment()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); } - return builder.build(); }; } @@ -175,5 +182,4 @@ private static Function setUriSchemeFunc(final String scheme) { public String getScheme() { return SCHEME; } - } diff --git a/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/EncryptedFileSystemTest.java b/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/EncryptedFileSystemTest.java index 73221092b..a1c09a0b8 100644 --- a/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/EncryptedFileSystemTest.java +++ b/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/EncryptedFileSystemTest.java @@ -28,21 +28,21 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.google.common.io.ByteStreams; import com.palantir.crypto2.cipher.AesCtrCipher; import com.palantir.crypto2.keys.KeyMaterial; import com.palantir.crypto2.keys.KeyStorageStrategy; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Arrays; import java.util.EnumSet; import java.util.Random; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -118,32 +118,30 @@ public void testDelegateStreamIsClosed() throws IOException { @Test public void testEncryptDecrypt_success() throws IllegalArgumentException, IOException { byte[] data = new byte[MB]; - byte[] readData = new byte[MB]; random.nextBytes(data); OutputStream os = efs.create(path); - IOUtils.write(data, os); + os.write(data); os.close(); // Read using EncryptedFileSystem yields input data InputStream is = efs.open(path); - IOUtils.readFully(is, readData); + byte[] readData = ByteStreams.toByteArray(is); is.close(); assertThat(data).isEqualTo(readData); // Read using delegate FileSystem does not yield input data is = delegateFs.open(path); - IOUtils.readFully(is, readData); + byte[] rawData = ByteStreams.toByteArray(is); is.close(); - assertThat(data).isNotEqualTo(readData); + assertThat(data).isNotEqualTo(rawData); } @Test public void testEncryptDecrypt_secondaryCreateMethod() throws IOException { byte[] data = new byte[MB]; - byte[] readData = new byte[MB]; random.nextBytes(data); OutputStream os = efs.create( @@ -155,38 +153,37 @@ public void testEncryptDecrypt_secondaryCreateMethod() throws IOException { 64 * 1024 * 1024, null, null); - IOUtils.write(data, os); + os.write(data); os.close(); // Read using EncryptedFileSystem yields input data InputStream is = efs.open(path); - IOUtils.readFully(is, readData); + byte[] readData = ByteStreams.toByteArray(is); is.close(); assertThat(data).isEqualTo(readData); // Read using delegate FileSystem does not yield input data is = delegateFs.open(path); - IOUtils.readFully(is, readData); + byte[] rawData = ByteStreams.toByteArray(is); is.close(); - assertThat(data).isNotEqualTo(readData); + assertThat(data).isNotEqualTo(rawData); } @Test public void testEncryptDecrypt_decryptSeek() throws IllegalArgumentException, IOException { byte[] data = new byte[MB]; int seekPos = MB / 2; - byte[] readData = new byte[MB - seekPos]; random.nextBytes(data); OutputStream os = efs.create(path); - IOUtils.write(data, os); + os.write(data); os.close(); FSDataInputStream is = efs.open(path); is.seek(seekPos); - IOUtils.readFully(is, readData); + byte[] readData = ByteStreams.toByteArray(is); byte[] actualReadData = Arrays.copyOfRange(data, seekPos, MB); assertThat(actualReadData).isEqualTo(readData); @@ -390,13 +387,12 @@ public void testAppend() { public void testCopyFromLocalFile() throws IOException { File file = folder.newFile(); byte[] data = "data".getBytes(StandardCharsets.UTF_8); - byte[] readBytes = new byte[data.length]; - IOUtils.write(data, new FileOutputStream(file)); + Files.write(file.toPath(), data); efs.copyFromLocalFile(new Path(file.getAbsolutePath()), path); FSDataInputStream input = efs.open(path); - IOUtils.readFully(input, readBytes); + byte[] readBytes = ByteStreams.toByteArray(input); assertThat(readBytes).isEqualTo(data); } diff --git a/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystemTest.java b/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystemTest.java index 538454207..c2390220b 100644 --- a/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystemTest.java +++ b/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/StandaloneEncryptedFileSystemTest.java @@ -21,7 +21,6 @@ import com.google.common.io.ByteStreams; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -31,7 +30,6 @@ import java.security.KeyPair; import java.util.Base64; import java.util.UUID; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; @@ -83,22 +81,21 @@ public void testGetUri_schemeIsCorrect() { @Test public void testReadWrite() throws IOException { - byte[] readData = new byte[DATA.length()]; - // Write encrypted data OutputStream os = efs.create(path); - IOUtils.write(DATA, os); + + os.write(DATA_BYTES); os.close(); // Read encrypted data InputStream dis = efs.open(path); - IOUtils.readFully(dis, readData); - assertThat(readData).containsExactly(DATA_BYTES); + byte[] encryptedData = ByteStreams.toByteArray(dis); + assertThat(encryptedData).containsExactly(DATA_BYTES); // Raw data is not the same dis = rawFs.open(path); - IOUtils.readFully(dis, readData); - assertThat(readData).isNotEqualTo(DATA_BYTES); + byte[] rawData = ByteStreams.toByteArray(dis); + assertThat(rawData).isNotEqualTo(DATA_BYTES); // KeyMaterial file exists assertThat(rawFs.exists(keyMaterialPath(path))).isTrue(); @@ -202,34 +199,34 @@ private Path writeData(File rootFolder) throws IOException { // Write encrypted data java.nio.file.Path filePath = Files.createTempFile(rootFolder.toPath(), "prefix", "suffix"); Path newPath = new Path(filePath.toAbsolutePath().toString()); - OutputStream os = efs.create(newPath); - IOUtils.write(DATA, os); - os.close(); + + try (OutputStream os = efs.create(newPath)) { + os.write(DATA_BYTES); + } return newPath; } @Test - public void testMakeQualified() throws IOException { + public void testMakeQualified() { assertThat(efs.makeQualified(pathWithScheme)).isEqualTo(pathWithScheme); } @Test public void testOnlyPublicKey() throws IOException { byte[] dataBytes = DATA.getBytes(StandardCharsets.UTF_8); - byte[] readData = new byte[DATA.length()]; conf.unset(StandaloneEncryptedFileSystem.PRIVATE_KEY_CONF); FileSystem efsPublic = FileSystem.newInstance(EFS_URI, conf); // Write encrypted data OutputStream os = efsPublic.create(path); - IOUtils.write(DATA, os); + os.write(DATA_BYTES); os.close(); // Raw data is not the same InputStream dis = rawFs.open(path); - IOUtils.readFully(dis, readData); + byte[] readData = ByteStreams.toByteArray(dis); assertThat(readData).isNotEqualTo(dataBytes); // KeyMaterial file exists @@ -242,14 +239,14 @@ public void testOnlyPublicKey() throws IOException { } @Test - public void testNoPublicKey() throws IOException { + public void testNoPublicKey() { assertThatExceptionOfType(NullPointerException.class) .isThrownBy(() -> FileSystem.newInstance(EFS_URI, getBaseConf())) .withMessage("Public Key must be configured for key %s", StandaloneEncryptedFileSystem.PUBLIC_KEY_CONF); } @Test - public void testBackingFsInvalid() throws IOException { + public void testBackingFsInvalid() { conf = getBaseConf(); conf.set("fs.nope.impl", StandaloneEncryptedFileSystem.class.getCanonicalName()); @@ -262,12 +259,12 @@ public void testBackingFsInvalid() throws IOException { public void testFileNameCollidesWithKeyMaterial() throws IOException { // Write encrypted data OutputStream os = efs.create(path); - IOUtils.write(DATA, os); + os.write(DATA_BYTES); os.close(); // Write encrypted data os = efs.create(keyMaterialPath(path)); - IOUtils.write(DATA, os); + os.write(DATA_BYTES); os.close(); // NOTE(jellis): IllegalArgumentException is the most likely exception, however if the first byte of the @@ -294,13 +291,12 @@ public void testListStatus_keyMaterialFilesFiltered() throws IOException { @Test // https://github.com/palantir/hadoop-crypto/issues/27 public void testCopyFromLocalFile() throws IOException { File file = folder.newFile(); - byte[] readBytes = new byte[DATA_BYTES.length]; - IOUtils.write(DATA_BYTES, new FileOutputStream(file)); + Files.write(file.toPath(), DATA_BYTES); efs.copyFromLocalFile(new Path(file.getAbsolutePath()), path); FSDataInputStream input = efs.open(path); - IOUtils.readFully(input, readBytes); + byte[] readBytes = ByteStreams.toByteArray(input); assertThat(readBytes).containsExactly(DATA_BYTES); } diff --git a/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/example/ExampleUsage.java b/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/example/ExampleUsage.java index de8798946..a4b9b4027 100644 --- a/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/example/ExampleUsage.java +++ b/hadoop-crypto/src/test/java/com/palantir/crypto2/hadoop/example/ExampleUsage.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import com.google.common.io.ByteStreams; import com.palantir.crypto2.hadoop.EncryptedFileSystem; import com.palantir.crypto2.hadoop.FileKeyStorageStrategy; import com.palantir.crypto2.hadoop.TestKeyPairs; @@ -29,7 +30,6 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.security.KeyPair; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -54,7 +54,6 @@ public void encryptedFileSystem_exampleUse() throws URISyntaxException, IOExcept // Init data and local path to write to byte[] data = "test".getBytes(StandardCharsets.UTF_8); - byte[] readData = new byte[data.length]; Path path = new Path(folder.newFile().getAbsolutePath()); // Write data out to the encrypted stream @@ -64,13 +63,13 @@ public void encryptedFileSystem_exampleUse() throws URISyntaxException, IOExcept // Reading through the decrypted stream produces the original bytes InputStream ein = efs.open(path); - IOUtils.readFully(ein, readData); + byte[] readData = ByteStreams.toByteArray(ein); assertThat(data).isEqualTo(readData); // Reading through the raw stream produces the encrypted bytes InputStream in = fs.open(path); - IOUtils.readFully(in, readData); - assertThat(data).isNotEqualTo(readData); + byte[] rawBytes = ByteStreams.toByteArray(in); + assertThat(data).isNotEqualTo(rawBytes); // Wrapped symmetric key is stored next to the encrypted file assertThat(fs.exists(new Path(path + FileKeyStorageStrategy.EXTENSION))).isTrue(); diff --git a/versions.lock b/versions.lock index 8312dcc22..8d3ecfda1 100644 --- a/versions.lock +++ b/versions.lock @@ -1,78 +1,24 @@ # Run ./gradlew --write-locks to regenerate this file -asm:asm:3.1 (1 constraints: 510c34f0) -com.google.code.findbugs:jsr305:3.0.2 (3 constraints: 5327ad58) -com.google.code.gson:gson:2.2.4 (1 constraints: 8c0d3f2f) +com.google.code.findbugs:jsr305:3.0.2 (3 constraints: 7c2a16b0) com.google.errorprone:error_prone_annotations:2.3.4 (1 constraints: 1b0af2b4) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) -com.google.guava:guava:30.0-jre (5 constraints: e841c185) +com.google.guava:guava:30.0-jre (1 constraints: 45069047) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) com.google.j2objc:j2objc-annotations:1.3 (1 constraints: b809eda0) -com.google.protobuf:protobuf-java:2.5.0 (1 constraints: 8b0d442f) -com.jamesmurty.utils:java-xmlbuilder:0.4 (1 constraints: e40aa5ca) -com.jcraft:jsch:0.1.42 (1 constraints: bb0ded3c) com.palantir.seek-io:seek-io:1.3.0 (1 constraints: 0605f935) -com.sun.jersey:jersey-core:1.9 (3 constraints: ff24a1f3) -com.sun.jersey:jersey-json:1.9 (1 constraints: 300d5f14) -com.sun.jersey:jersey-server:1.9 (1 constraints: 300d5f14) -com.sun.xml.bind:jaxb-impl:2.2.3-1 (1 constraints: 330c2404) -com.thoughtworks.paranamer:paranamer:2.3 (1 constraints: ef08468a) -commons-beanutils:commons-beanutils:1.7.0 (1 constraints: da0e635f) -commons-beanutils:commons-beanutils-core:1.8.0 (1 constraints: 1d134124) -commons-cli:commons-cli:1.2 (1 constraints: 290d5814) -commons-codec:commons-codec:1.15 (6 constraints: 7349809b) -commons-collections:commons-collections:3.2.2 (3 constraints: ac259154) -commons-configuration:commons-configuration:1.6 (1 constraints: 2d0d5c14) -commons-digester:commons-digester:1.8 (1 constraints: bf1228fe) -commons-httpclient:commons-httpclient:3.1 (1 constraints: 2a0d5d14) -commons-io:commons-io:2.4 (1 constraints: 2c0d5d14) -commons-lang:commons-lang:2.6 (3 constraints: 94247ea1) -commons-logging:commons-logging:1.2 (9 constraints: a08097cf) -commons-net:commons-net:3.1 (1 constraints: 2a0d5d14) -io.netty:netty:3.7.0.Final (1 constraints: a20f0480) -javax.activation:activation:1.1 (1 constraints: e609cdaa) -javax.servlet:servlet-api:2.5 (1 constraints: 2d0d5e14) -javax.servlet.jsp:jsp-api:2.1 (1 constraints: 290d5a14) -javax.xml.bind:jaxb-api:2.2.2 (1 constraints: 690bc8e3) -javax.xml.stream:stax-api:1.0-2 (1 constraints: 440a22bf) -jline:jline:0.9.94 (1 constraints: c60d0036) -log4j:log4j:1.2.17 (5 constraints: 5337a618) -net.java.dev.jets3t:jets3t:0.9.0 (1 constraints: 8d0d462f) +commons-logging:commons-logging:1.1.3 (1 constraints: b0102eb8) net.java.dev.jna:jna:5.5.0 (1 constraints: a80ee955) net.sf.jopt-simple:jopt-simple:4.6 (1 constraints: 610a91b7) -org.apache.avro:avro:1.7.4 (1 constraints: 900d492f) -org.apache.commons:commons-compress:1.4.1 (2 constraints: d7161240) org.apache.commons:commons-crypto:1.1.0 (1 constraints: 0405f335) -org.apache.commons:commons-math3:3.6.1 (3 constraints: ef1c9626) -org.apache.curator:curator-client:2.7.1 (2 constraints: 6a1d2734) -org.apache.curator:curator-framework:2.7.1 (2 constraints: b01bbfaf) -org.apache.curator:curator-recipes:2.7.1 (1 constraints: 8e0d4b2f) -org.apache.directory.api:api-asn1-api:1.0.0-M20 (1 constraints: 3d163b13) -org.apache.directory.api:api-util:1.0.0-M20 (1 constraints: 3d163b13) -org.apache.directory.server:apacheds-i18n:2.0.0-M15 (1 constraints: 42164713) -org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15 (1 constraints: 8f0d3b45) -org.apache.hadoop:hadoop-annotations:2.7.3 (1 constraints: 900d4d2f) -org.apache.hadoop:hadoop-auth:2.7.3 (1 constraints: 900d4d2f) -org.apache.hadoop:hadoop-common:2.7.3 (1 constraints: 0e050d36) -org.apache.htrace:htrace-core:3.1.0-incubating (1 constraints: d911c6dc) -org.apache.httpcomponents:httpclient:4.5.13 (3 constraints: 3a1db857) -org.apache.httpcomponents:httpcore:4.4.13 (3 constraints: da20366d) -org.apache.zookeeper:zookeeper:3.4.6 (5 constraints: b5473b06) +org.apache.commons:commons-math3:3.6.1 (2 constraints: 670ffb89) +org.apache.hadoop:hadoop-client-api:3.3.0 (2 constraints: b8159ddf) +org.apache.hadoop:hadoop-client-runtime:3.3.0 (1 constraints: 08050336) +org.apache.htrace:htrace-core4:4.1.0-incubating (1 constraints: 01158a88) org.checkerframework:checker-qual:3.5.0 (1 constraints: 1a0af9b4) -org.codehaus.jackson:jackson-core-asl:1.9.13 (6 constraints: 7e50faca) -org.codehaus.jackson:jackson-jaxrs:1.8.3 (1 constraints: da0bedeb) -org.codehaus.jackson:jackson-mapper-asl:1.9.13 (5 constraints: 8e3fb0c8) -org.codehaus.jackson:jackson-xc:1.8.3 (1 constraints: da0bedeb) -org.codehaus.jettison:jettison:1.1 (1 constraints: 720b65d4) org.immutables:value:2.8.8 (1 constraints: 14051536) -org.mortbay.jetty:jetty:6.1.26 (1 constraints: c30d113d) -org.mortbay.jetty:jetty-util:6.1.26 (2 constraints: d51883d6) org.openjdk.jmh:jmh-core:1.22 (2 constraints: eb1529d8) org.openjdk.jmh:jmh-generator-annprocess:1.22 (1 constraints: d904f530) -org.slf4j:slf4j-api:1.7.30 (11 constraints: 38933acd) -org.slf4j:slf4j-log4j12:1.7.30 (3 constraints: 292803a2) -org.tukaani:xz:1.0 (1 constraints: 0c0f2959) -org.xerial.snappy:snappy-java:1.0.4.1 (1 constraints: ac09cdaf) -xmlenc:xmlenc:0.52 (1 constraints: 5d0db521) +org.slf4j:slf4j-api:1.7.30 (2 constraints: 26160805) [Test dependencies] com.google.code.findbugs:annotations:3.0.1 (1 constraints: 9e0aafc3) diff --git a/versions.props b/versions.props index 88d2a8686..e821c34db 100644 --- a/versions.props +++ b/versions.props @@ -3,7 +3,7 @@ com.palantir.seek-io:seek-io = 1.3.0 junit:junit = 4.13.1 org.apache.commons:commons-crypto = 1.1.0 org.apache.commons:commons-math3 = 3.6.1 -org.apache.hadoop:hadoop-common = 2.7.3 +org.apache.hadoop:hadoop-* = 3.3.0 org.assertj:assertj-* = 3.16.1 org.hamcrest:hamcrest-* = 1.3 org.immutables:value = 2.8.8 @@ -12,13 +12,3 @@ org.mockito:mockito-core = 3.5.15 org.openjdk.jmh:* = 1.22 org.ow2.asm:asm = 9.0 org.slf4j:slf4j-* = 1.7.30 - -# Hadoop dependencies -commons-codec:commons-codec = 1.15 -commons-collections:commons-collections = 3.2.2 -commons-lang:commons-lang = 2.6 -commons-logging:commons-logging = 1.2 -log4j:log4j = 1.2.17 -org.apache.httpcomponents:httpclient = 4.5.13 -org.apache.httpcomponents:httpcore = 4.4.13 -org.codehaus.jackson:jackson-*-asl = 1.9.13