From 04f1937cfe02d20a333aa0b0daeff609f963acec Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 13 Sep 2023 12:57:58 +0100 Subject: [PATCH 1/5] HADOOP-18894: upgrade sshd-core due to CVEs --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index ae5c434afdcf7..d6b71e1836fd4 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1133,7 +1133,7 @@ org.apache.sshd sshd-core - 1.6.0 + 2.7.0 org.apache.ftpserver From 66325b2f2cc7449db46b6ce129b461bf71d57a59 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 13 Sep 2023 15:21:03 +0100 Subject: [PATCH 2/5] fix compile issues --- hadoop-common-project/hadoop-common/pom.xml | 5 +++++ .../hadoop/fs/contract/sftp/SFTPContract.java | 7 +++---- .../apache/hadoop/fs/sftp/TestSFTPFileSystem.java | 14 +++++--------- hadoop-project/pom.xml | 8 +++++++- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 448f644b6d018..3b3279ff4e25f 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -316,6 +316,11 @@ sshd-core test + + org.apache.sshd + sshd-sftp + test + org.apache.ftpserver ftpserver-core diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/sftp/SFTPContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/sftp/SFTPContract.java index f72a2aec86242..631c89586514a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/sftp/SFTPContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/sftp/SFTPContract.java @@ -31,12 +31,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.sftp.SFTPFileSystem; -import org.apache.sshd.common.NamedFactory; import org.apache.sshd.server.SshServer; -import org.apache.sshd.server.auth.UserAuth; +import org.apache.sshd.server.auth.UserAuthFactory; import org.apache.sshd.server.auth.password.UserAuthPasswordFactory; import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; -import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory; +import org.apache.sshd.sftp.server.SftpSubsystemFactory; public class SFTPContract extends AbstractFSContract { @@ -61,7 +60,7 @@ public void init() throws IOException { sshd.setPort(0); sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider()); - List> userAuthFactories = new ArrayList<>(); + List userAuthFactories = new ArrayList<>(); userAuthFactories.add(new UserAuthPasswordFactory()); sshd.setUserAuthFactories(userAuthFactories); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java index e8ba5f211eb8d..dffaabe5778ef 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java @@ -22,7 +22,7 @@ import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -35,16 +35,13 @@ import org.apache.hadoop.test.GenericTestUtils; import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; -import org.apache.sshd.common.NamedFactory; -import org.apache.sshd.server.Command; import org.apache.sshd.server.SshServer; -import org.apache.sshd.server.auth.UserAuth; +import org.apache.sshd.server.auth.UserAuthFactory; import org.apache.sshd.server.auth.password.PasswordAuthenticator; import org.apache.sshd.server.auth.password.UserAuthPasswordFactory; import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; import org.apache.sshd.server.session.ServerSession; -import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory; - +import org.apache.sshd.server.subsystem.SubsystemFactory; import org.junit.After; import org.junit.AfterClass; @@ -82,8 +79,7 @@ private static void startSshdServer() throws IOException { sshd.setPort(0); sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider()); - List> userAuthFactories = - new ArrayList>(); + List userAuthFactories = new ArrayList<>(); userAuthFactories.add(new UserAuthPasswordFactory()); sshd.setUserAuthFactories(userAuthFactories); @@ -100,7 +96,7 @@ public boolean authenticate(String username, String password, }); sshd.setSubsystemFactories( - Arrays.>asList(new SftpSubsystemFactory())); + Collections.singletonList(new org.apache.sshd.sftp.server.SftpSubsystemFactory())); sshd.start(); port = sshd.getPort(); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index d6b71e1836fd4..e85a5a0906cbd 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -202,6 +202,7 @@ 1.5.4 2.0 + 2.10.0 1.7.1 2.2.4 4.13.2 @@ -1133,7 +1134,12 @@ org.apache.sshd sshd-core - 2.7.0 + ${sshd.version} + + + org.apache.sshd + sshd-sftp + ${sshd.version} org.apache.ftpserver From 71c6db7a38f12bc6b6a5bf618bb326af2e3ccf38 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 13 Sep 2023 17:15:10 +0100 Subject: [PATCH 3/5] fix imports --- .../org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java index dffaabe5778ef..e425c2dea284a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java @@ -41,9 +41,7 @@ import org.apache.sshd.server.auth.password.UserAuthPasswordFactory; import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; import org.apache.sshd.server.session.ServerSession; -import org.apache.sshd.server.subsystem.SubsystemFactory; -import org.junit.After; -import org.junit.AfterClass; +import org.apache.sshd.sftp.server.SftpSubsystemFactory; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertArrayEquals; @@ -51,6 +49,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -96,7 +96,7 @@ public boolean authenticate(String username, String password, }); sshd.setSubsystemFactories( - Collections.singletonList(new org.apache.sshd.sftp.server.SftpSubsystemFactory())); + Collections.singletonList(new SftpSubsystemFactory())); sshd.start(); port = sshd.getPort(); From f3a7a86e6f75bdc18443c903c6d63421c8d564e8 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Mon, 15 Jan 2024 14:54:16 +0100 Subject: [PATCH 4/5] use sshd 2.11.0 --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e85a5a0906cbd..e2fe8846aed34 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -202,7 +202,7 @@ 1.5.4 2.0 - 2.10.0 + 2.11.0 1.7.1 2.2.4 4.13.2 From 9b5dcca518edbb9b886900f715a65c9d81dfccb6 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Mon, 15 Jan 2024 15:09:40 +0100 Subject: [PATCH 5/5] Update LICENSE-binary --- LICENSE-binary | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LICENSE-binary b/LICENSE-binary index 1ebc44b0580a3..3720a78095635 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -335,6 +335,9 @@ org.apache.kerby:kerby-pkix:2.0.3 org.apache.kerby:kerby-util:2.0.3 org.apache.kerby:kerby-xdr:2.0.3 org.apache.kerby:token-provider:2.0.3 +org.apache.sshd:sshd-common:2.11.0 +org.apache.sshd:sshd-core:2.11.0 +org.apache.sshd:sshd-sftp:2.11.0 org.apache.solr:solr-solrj:8.11.2 org.apache.yetus:audience-annotations:0.5.0 org.apache.zookeeper:zookeeper:3.8.3