From ea919aa4be0fea8061b2594e1046cd1aca9512c4 Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Fri, 10 Mar 2017 14:21:23 -0500 Subject: [PATCH] GH-198: (S)FTP Server: Use an OS-selected Port Fixes spring-projects/spring-integration-samples#198 The `SocketUtils.findAvailableServerSocket()` isn't reliable for port selection and its `socket.close()` may cause a port selection by some other process. Use OS-selected port for SshdServer in SFTP sample Revert unexpected refactoring after renaming properties Upgrade to SSHD-1.4, FTP Server 1.1, fix tests and Boot 2.0 compatibility --- .../integration/samples/ftp/TestSuite.java | 34 ++++--- .../integration/samples/jpa/Main.java | 66 ++++++++------ .../samples/sftp/EmbeddedSftpServer.java | 62 +++++++------ .../samples/sftp/SftpTestUtils.java | 91 +++++++------------ .../SftpInboundReceiveSample-context.xml | 14 --- .../SftpOutboundGatewaySample-context.xml | 14 --- .../SftpOutboundTransferSample-context.xml | 14 --- .../spring/integration/SftpSampleCommon.xml | 17 +++- basic/sftp/src/test/resources/user.properties | 12 +-- .../websocket/standard/ApplicationTests.java | 4 +- build.gradle | 4 +- 11 files changed, 148 insertions(+), 184 deletions(-) diff --git a/basic/ftp/src/test/java/org/springframework/integration/samples/ftp/TestSuite.java b/basic/ftp/src/test/java/org/springframework/integration/samples/ftp/TestSuite.java index 132c7e8a..d48d813d 100644 --- a/basic/ftp/src/test/java/org/springframework/integration/samples/ftp/TestSuite.java +++ b/basic/ftp/src/test/java/org/springframework/integration/samples/ftp/TestSuite.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.springframework.integration.samples.ftp; import java.io.File; import java.io.IOException; -import java.net.SocketException; import org.apache.commons.io.FileUtils; import org.apache.ftpserver.FtpServer; import org.apache.ftpserver.FtpServerFactory; import org.apache.ftpserver.ftplet.FtpException; +import org.apache.ftpserver.listener.Listener; import org.apache.ftpserver.listener.ListenerFactory; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -34,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.integration.samples.ftp.support.TestUserManager; -import org.springframework.integration.test.util.SocketUtils; /** * Test Suite that will bootstrap an embedded Apache FTP Server. Additionally some @@ -46,16 +46,18 @@ import org.springframework.integration.test.util.SocketUtils; */ @RunWith(Suite.class) @Suite.SuiteClasses({ - FtpOutboundChannelAdapterSample.class, - FtpInboundChannelAdapterSample.class, - FtpOutboundGatewaySample.class - }) + FtpOutboundChannelAdapterSample.class, + FtpInboundChannelAdapterSample.class, + FtpOutboundGatewaySample.class +}) public class TestSuite { private static final Logger LOGGER = LoggerFactory.getLogger(TestSuite.class); - public static final String FTP_ROOT_DIR = "target" + File.separator + "ftproot"; + public static final String FTP_ROOT_DIR = "target" + File.separator + "ftproot"; + public static final String LOCAL_FTP_TEMP_DIR = "target" + File.separator + "local-ftp-temp"; + public static final String SERVER_PORT_SYSTEM_PROPERTY = "availableServerPort"; @ClassRule @@ -64,20 +66,18 @@ public class TestSuite { public static FtpServer server; @BeforeClass - public static void setupFtpServer() throws FtpException, SocketException, IOException { + public static void setupFtpServer() throws FtpException, IOException { - final int availableServerSocket; + Integer availableServerSocket; if (System.getProperty(SERVER_PORT_SYSTEM_PROPERTY) == null) { - availableServerSocket = SocketUtils.findAvailableServerSocket(4444); - System.setProperty(SERVER_PORT_SYSTEM_PROPERTY, Integer.valueOf(availableServerSocket).toString()); - } else { + availableServerSocket = 0; + } + else { availableServerSocket = Integer.valueOf(System.getProperty(SERVER_PORT_SYSTEM_PROPERTY)); } - LOGGER.info("Using open server port..." + availableServerSocket); - - File ftpRoot = new File (FTP_ROOT_DIR); + File ftpRoot = new File(FTP_ROOT_DIR); ftpRoot.mkdirs(); TestUserManager userManager = new TestUserManager(ftpRoot.getAbsolutePath()); @@ -94,6 +94,10 @@ public class TestSuite { server.start(); + Listener listener = serverFactory.getListeners().values().iterator().next(); + availableServerSocket = listener.getPort(); + LOGGER.info("Using open server port..." + availableServerSocket); + System.setProperty(SERVER_PORT_SYSTEM_PROPERTY, availableServerSocket.toString()); } @AfterClass diff --git a/basic/jpa/src/main/java/org/springframework/integration/samples/jpa/Main.java b/basic/jpa/src/main/java/org/springframework/integration/samples/jpa/Main.java index f1335e1c..511adfd9 100644 --- a/basic/jpa/src/main/java/org/springframework/integration/samples/jpa/Main.java +++ b/basic/jpa/src/main/java/org/springframework/integration/samples/jpa/Main.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.text.SimpleDateFormat; import java.util.List; import java.util.Scanner; +import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.boot.builder.SpringApplicationBuilder; @@ -55,13 +56,13 @@ public class Main { final Scanner scanner = new Scanner(System.in); System.out.println("\n=========================================================" - + "\n " - + "\n Welcome to the Spring Integration JPA Sample! " - + "\n " - + "\n For more information please visit: " - + "\n http://www.springintegration.org/ " - + "\n " - + "\n=========================================================" ); + + "\n " + + "\n Welcome to the Spring Integration JPA Sample! " + + "\n " + + "\n For more information please visit: " + + "\n http://www.springintegration.org/ " + + "\n " + + "\n========================================================="); System.out.println("Please enter a choice and press : "); System.out.println("\t1. Use Hibernate"); @@ -70,21 +71,25 @@ public class Main { System.out.println("\tq. Quit the application"); System.out.print("Enter you choice: "); - SpringApplicationBuilder springApplicationBuilder = new SpringApplicationBuilder(Main.class).web(false); + SpringApplicationBuilder springApplicationBuilder = new SpringApplicationBuilder(Main.class) + .web(WebApplicationType.NONE); while (true) { final String input = scanner.nextLine(); - if("1".equals(input.trim())) { + if ("1".equals(input.trim())) { springApplicationBuilder.sources(HibernateJpaAutoConfiguration.class); break; - } else if("2".equals(input.trim())) { + } + else if ("2".equals(input.trim())) { springApplicationBuilder.profiles("eclipseLink"); break; - } else if("q".equals(input.trim())) { + } + else if ("q".equals(input.trim())) { System.out.println("Exiting application...bye."); System.exit(0); - } else { + } + else { System.out.println("Invalid choice\n\n"); System.out.print("Enter you choice: "); } @@ -129,12 +134,12 @@ public class Main { } - private static void createPersonDetails(final Scanner scanner,PersonService service) { - while(true) { + private static void createPersonDetails(final Scanner scanner, PersonService service) { + while (true) { System.out.print("\nEnter the Person's name:"); String name = null; - while(true) { + while (true) { name = scanner.nextLine(); @@ -151,9 +156,9 @@ public class Main { person = service.createPerson(person); System.out.println("Created person record with id: " + person.getId()); System.out.print("Do you want to create another person? (y/n)"); - String choice = scanner.nextLine(); + String choice = scanner.nextLine(); - if(!"y".equalsIgnoreCase(choice)) { + if (!"y".equalsIgnoreCase(choice)) { break; } } @@ -161,22 +166,23 @@ public class Main { private static void findPeople(final PersonService service) { - System.out.println("ID NAME CREATED"); - System.out.println("=================================="); + System.out.println("ID NAME CREATED"); + System.out.println("=================================="); - final List people = service.findPeople(); + final List people = service.findPeople(); - if(people != null && !people.isEmpty()) { - for(Person person : people) { - System.out.print(String.format("%d, %s, ", person.getId(), person.getName())); - System.out.println(DATE_FORMAT.format(person.getCreatedDateTime()));//NOSONAR - } - } else { - System.out.println( - String.format("No Person record found.")); + if (people != null && !people.isEmpty()) { + for (Person person : people) { + System.out.print(String.format("%d, %s, ", person.getId(), person.getName())); + System.out.println(DATE_FORMAT.format(person.getCreatedDateTime()));//NOSONAR } + } + else { + System.out.println( + String.format("No Person record found.")); + } - System.out.println("==================================\n\n"); + System.out.println("==================================\n\n"); } } diff --git a/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/EmbeddedSftpServer.java b/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/EmbeddedSftpServer.java index 2b85e657..1bf41866 100644 --- a/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/EmbeddedSftpServer.java +++ b/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/EmbeddedSftpServer.java @@ -1,5 +1,5 @@ /* - * Copyright 2014-2015 the original author or authors. + * Copyright 2014-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,30 +16,28 @@ package org.springframework.integration.samples.sftp; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.file.Paths; import java.security.KeyFactory; import java.security.PublicKey; import java.security.spec.RSAPublicKeySpec; +import java.util.Arrays; import java.util.Collections; -import org.apache.sshd.SshServer; -import org.apache.sshd.common.NamedFactory; import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory; -import org.apache.sshd.common.util.Base64; -import org.apache.sshd.server.Command; -import org.apache.sshd.server.PublickeyAuthenticator; +import org.apache.sshd.server.SshServer; import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; -import org.apache.sshd.server.session.ServerSession; -import org.apache.sshd.server.sftp.SftpSubsystem; +import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.context.SmartLifecycle; import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.FileSystemResource; -import org.springframework.util.SocketUtils; +import org.springframework.integration.sftp.session.DefaultSftpSessionFactory; +import org.springframework.util.Base64Utils; import org.springframework.util.StreamUtils; /** @@ -47,7 +45,10 @@ import org.springframework.util.StreamUtils; */ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle { - public static final int PORT = SocketUtils.findAvailableTcpPort(); + /** + * Let OS to obtain the proper port + */ + public static final int PORT = 0; private final SshServer server = SshServer.setUpDefaultServer(); @@ -55,31 +56,36 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle { private volatile boolean running; + private DefaultSftpSessionFactory defaultSftpSessionFactory; + public void setPort(int port) { this.port = port; } + public void setDefaultSftpSessionFactory(DefaultSftpSessionFactory defaultSftpSessionFactory) { + this.defaultSftpSessionFactory = defaultSftpSessionFactory; + } + @Override public void afterPropertiesSet() throws Exception { final PublicKey allowedKey = decodePublicKey(); - this.server.setPublickeyAuthenticator(new PublickeyAuthenticator() { - - @Override - public boolean authenticate(String username, PublicKey key, ServerSession session) { - return key.equals(allowedKey); - } - - }); + this.server.setPublickeyAuthenticator((username, key, session) -> key.equals(allowedKey)); this.server.setPort(this.port); - this.server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider("hostkey.ser")); - this.server.setSubsystemFactories(Collections.>singletonList(new SftpSubsystem.Factory())); - final String virtualDir = new FileSystemResource("").getFile().getAbsolutePath(); - server.setFileSystemFactory(new VirtualFileSystemFactory(virtualDir)); + this.server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(new File("hostkey.ser"))); + this.server.setSubsystemFactories(Collections.singletonList(new SftpSubsystemFactory())); + final String pathname = System.getProperty("java.io.tmpdir") + File.separator + "sftptest" + File.separator; + new File(pathname).mkdirs(); + server.setFileSystemFactory(new VirtualFileSystemFactory(Paths.get(pathname))); } private PublicKey decodePublicKey() throws Exception { InputStream stream = new ClassPathResource("META-INF/keys/sftp_rsa.pub").getInputStream(); - byte[] decodeBuffer = Base64.decodeBase64(StreamUtils.copyToByteArray(stream)); + byte[] keyBytes = StreamUtils.copyToByteArray(stream); + // strip any newline chars + while (keyBytes[keyBytes.length - 1] == 0x0a || keyBytes[keyBytes.length - 1] == 0x0d) { + keyBytes = Arrays.copyOf(keyBytes, keyBytes.length - 1); + } + byte[] decodeBuffer = Base64Utils.decode(keyBytes); ByteBuffer bb = ByteBuffer.wrap(decodeBuffer); int len = bb.getInt(); byte[] type = new byte[len]; @@ -116,8 +122,9 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle { @Override public void start() { try { - server.start(); - this.running = true; + this.server.start(); + this.defaultSftpSessionFactory.setPort(this.server.getPort()); + this.running = true; } catch (IOException e) { throw new IllegalStateException(e); @@ -136,7 +143,7 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle { try { server.stop(true); } - catch (InterruptedException e) { + catch (Exception e) { throw new IllegalStateException(e); } finally { @@ -149,5 +156,4 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle { public boolean isRunning() { return this.running; } - } diff --git a/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/SftpTestUtils.java b/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/SftpTestUtils.java index c1a1d4c0..a99e9b89 100644 --- a/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/SftpTestUtils.java +++ b/basic/sftp/src/test/java/org/springframework/integration/samples/sftp/SftpTestUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 the original author or authors. + * Copyright 2014-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,19 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.springframework.integration.samples.sftp; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.IOException; -import org.springframework.beans.DirectFieldAccessor; import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.SessionCallback; -import org.springframework.integration.file.remote.session.Session; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.ChannelSftp.LsEntry; @@ -34,6 +32,8 @@ import com.jcraft.jsch.SftpException; /** * @author Gary Russell + * @author Artem Bilan + * * @since 4.1 * */ @@ -42,78 +42,57 @@ public class SftpTestUtils { public static void createTestFiles(RemoteFileTemplate template, final String... fileNames) { if (template != null) { final ByteArrayInputStream stream = new ByteArrayInputStream("foo".getBytes()); - template.execute(new SessionCallback() { - - @Override - public Void doInSession(Session session) throws IOException { - try { - session.mkdir("si.sftp.sample"); - } - catch (Exception e) { - assertThat(e.getMessage(), containsString("failed to create")); - } - for (int i = 0; i < fileNames.length; i++) { - stream.reset(); - session.write(stream, "si.sftp.sample/" + fileNames[i]); - } - return null; + template.execute((SessionCallback) session -> { + try { + session.mkdir("si.sftp.sample"); } + catch (Exception e) { + assertThat(e.getMessage(), containsString("failed to create")); + } + for (int i = 0; i < fileNames.length; i++) { + stream.reset(); + session.write(stream, "si.sftp.sample/" + fileNames[i]); + } + return null; }); } } public static void cleanUp(RemoteFileTemplate template, final String... fileNames) { if (template != null) { - template.execute(new SessionCallback() { - - @Override - public Void doInSession(Session session) throws IOException { - // TODO: avoid DFAs with Spring 4.1 (INT-3412) - ChannelSftp channel = (ChannelSftp) new DirectFieldAccessor(new DirectFieldAccessor(session) - .getPropertyValue("targetSession")).getPropertyValue("channel"); - for (int i = 0; i < fileNames.length; i++) { - try { - session.remove("si.sftp.sample/" + fileNames[i]); - } - catch (IOException e) {} - } + template.execute((SessionCallback) session -> { + for (int i = 0; i < fileNames.length; i++) { try { - // should be empty - channel.rmdir("si.sftp.sample"); + session.remove("si.sftp.sample/" + fileNames[i]); } - catch (SftpException e) { - fail("Expected remote directory to be empty " + e.getMessage()); - } - return null; + catch (IOException e) {} } + + // should be empty + session.rmdir("si.sftp.sample"); + return null; }); } } public static boolean fileExists(RemoteFileTemplate template, final String... fileNames) { if (template != null) { - return template.execute(new SessionCallback() { - - @Override - public Boolean doInSession(Session session) throws IOException { - // TODO: avoid DFAs with Spring 4.1 (INT-3412) - ChannelSftp channel = (ChannelSftp) new DirectFieldAccessor(new DirectFieldAccessor(session) - .getPropertyValue("targetSession")).getPropertyValue("channel"); - for (int i = 0; i < fileNames.length; i++) { - try { - SftpATTRS stat = channel.stat("si.sftp.sample/" + fileNames[i]); - if (stat == null) { - System.out.println("stat returned null for " + fileNames[i]); - return false; - } - } - catch (SftpException e) { - System.out.println("Remote file not present: " + e.getMessage() + ": " + fileNames[i]); + return template.execute(session -> { + ChannelSftp channel = (ChannelSftp) session.getClientInstance(); + for (int i = 0; i < fileNames.length; i++) { + try { + SftpATTRS stat = channel.stat("si.sftp.sample/" + fileNames[i]); + if (stat == null) { + System.out.println("stat returned null for " + fileNames[i]); return false; } } - return true; + catch (SftpException e) { + System.out.println("Remote file not present: " + e.getMessage() + ": " + fileNames[i]); + return false; + } } + return true; }); } else { diff --git a/basic/sftp/src/test/resources/META-INF/spring/integration/SftpInboundReceiveSample-context.xml b/basic/sftp/src/test/resources/META-INF/spring/integration/SftpInboundReceiveSample-context.xml index eb71e674..12c54208 100644 --- a/basic/sftp/src/test/resources/META-INF/spring/integration/SftpInboundReceiveSample-context.xml +++ b/basic/sftp/src/test/resources/META-INF/spring/integration/SftpInboundReceiveSample-context.xml @@ -9,20 +9,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - + + diff --git a/basic/sftp/src/test/resources/user.properties b/basic/sftp/src/test/resources/user.properties index ece0bb29..e020c4b1 100644 --- a/basic/sftp/src/test/resources/user.properties +++ b/basic/sftp/src/test/resources/user.properties @@ -1,7 +1,7 @@ -host=localhost -# -1 means the embedded Apache MINA SshServer. Change it to any real port, if you are going to test sample against real SFTP Server -port=-1 -username=user -passphrase=password +sftp.host=localhost +# 0 means the embedded Apache MINA SshServer. Change it to any real port, if you are going to test sample against real SFTP Server +sftp.port=0 +sftp.username=user +sftp.passphrase=password #private.keyfile=file:/home/someuser/.ssh/id_rsa -private.keyfile=classpath:META-INF/keys/sftp_rsa +sftp.private.keyfile=classpath:META-INF/keys/sftp_rsa diff --git a/basic/web-sockets/src/test/java/org/springframework/integration/samples/websocket/standard/ApplicationTests.java b/basic/web-sockets/src/test/java/org/springframework/integration/samples/websocket/standard/ApplicationTests.java index e6656b0c..3523309e 100644 --- a/basic/web-sockets/src/test/java/org/springframework/integration/samples/websocket/standard/ApplicationTests.java +++ b/basic/web-sockets/src/test/java/org/springframework/integration/samples/websocket/standard/ApplicationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 the original author or authors. + * Copyright 2014-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,8 +31,8 @@ import java.util.concurrent.TimeUnit; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.boot.context.embedded.LocalServerPort; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.server.LocalServerPort; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.integration.channel.DirectChannel; diff --git a/build.gradle b/build.gradle index 652d3071..4aa85468 100644 --- a/build.gradle +++ b/build.gradle @@ -157,7 +157,7 @@ subprojects { subproject -> ext { activeMqVersion = '5.13.4' - apacheSshdVersion = '0.14.0' + apacheSshdVersion = '1.4.0' aspectjVersion = '1.8.9' commonsDigesterVersion = '2.0' commonsDbcpVersion = '1.2.2' @@ -171,7 +171,7 @@ subprojects { subproject -> hamcrestVersion = '1.3' hibernateVersion = '5.0.9.Final' hibernateValidatorVersion = '4.2.0.Final' - ftpServerVersion = '1.0.6' + ftpServerVersion = '1.1.0' flexjsonVersion = '2.0' guavaVersion = '16.0.1' groovyVersion = '2.3.0'