GH-198: (S)FTP Server: Use an OS-selected Port

Fixes spring-projects/spring-integration-samples#198

The `SocketUtils.findAvailableServerSocket()` isn't reliable for port selection and its `socket.close()` may cause a port selection by some other process.

Use OS-selected port for SshdServer in SFTP sample

Revert unexpected refactoring after renaming properties

Upgrade to SSHD-1.4, FTP Server 1.1,  fix tests and Boot 2.0 compatibility
This commit is contained in:
Artem Bilan
2017-03-10 14:21:23 -05:00
committed by Gary Russell
parent a198e45239
commit ea919aa4be
11 changed files with 148 additions and 184 deletions

View File

@@ -13,16 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.samples.ftp;
import java.io.File;
import java.io.IOException;
import java.net.SocketException;
import org.apache.commons.io.FileUtils;
import org.apache.ftpserver.FtpServer;
import org.apache.ftpserver.FtpServerFactory;
import org.apache.ftpserver.ftplet.FtpException;
import org.apache.ftpserver.listener.Listener;
import org.apache.ftpserver.listener.ListenerFactory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -34,7 +35,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.integration.samples.ftp.support.TestUserManager;
import org.springframework.integration.test.util.SocketUtils;
/**
* Test Suite that will bootstrap an embedded Apache FTP Server. Additionally some
@@ -46,16 +46,18 @@ import org.springframework.integration.test.util.SocketUtils;
*/
@RunWith(Suite.class)
@Suite.SuiteClasses({
FtpOutboundChannelAdapterSample.class,
FtpInboundChannelAdapterSample.class,
FtpOutboundGatewaySample.class
})
FtpOutboundChannelAdapterSample.class,
FtpInboundChannelAdapterSample.class,
FtpOutboundGatewaySample.class
})
public class TestSuite {
private static final Logger LOGGER = LoggerFactory.getLogger(TestSuite.class);
public static final String FTP_ROOT_DIR = "target" + File.separator + "ftproot";
public static final String FTP_ROOT_DIR = "target" + File.separator + "ftproot";
public static final String LOCAL_FTP_TEMP_DIR = "target" + File.separator + "local-ftp-temp";
public static final String SERVER_PORT_SYSTEM_PROPERTY = "availableServerPort";
@ClassRule
@@ -64,20 +66,18 @@ public class TestSuite {
public static FtpServer server;
@BeforeClass
public static void setupFtpServer() throws FtpException, SocketException, IOException {
public static void setupFtpServer() throws FtpException, IOException {
final int availableServerSocket;
Integer availableServerSocket;
if (System.getProperty(SERVER_PORT_SYSTEM_PROPERTY) == null) {
availableServerSocket = SocketUtils.findAvailableServerSocket(4444);
System.setProperty(SERVER_PORT_SYSTEM_PROPERTY, Integer.valueOf(availableServerSocket).toString());
} else {
availableServerSocket = 0;
}
else {
availableServerSocket = Integer.valueOf(System.getProperty(SERVER_PORT_SYSTEM_PROPERTY));
}
LOGGER.info("Using open server port..." + availableServerSocket);
File ftpRoot = new File (FTP_ROOT_DIR);
File ftpRoot = new File(FTP_ROOT_DIR);
ftpRoot.mkdirs();
TestUserManager userManager = new TestUserManager(ftpRoot.getAbsolutePath());
@@ -94,6 +94,10 @@ public class TestSuite {
server.start();
Listener listener = serverFactory.getListeners().values().iterator().next();
availableServerSocket = listener.getPort();
LOGGER.info("Using open server port..." + availableServerSocket);
System.setProperty(SERVER_PORT_SYSTEM_PROPERTY, availableServerSocket.toString());
}
@AfterClass

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2015 the original author or authors.
* Copyright 2002-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Scanner;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
import org.springframework.boot.builder.SpringApplicationBuilder;
@@ -55,13 +56,13 @@ public class Main {
final Scanner scanner = new Scanner(System.in);
System.out.println("\n========================================================="
+ "\n "
+ "\n Welcome to the Spring Integration JPA Sample! "
+ "\n "
+ "\n For more information please visit: "
+ "\n http://www.springintegration.org/ "
+ "\n "
+ "\n=========================================================" );
+ "\n "
+ "\n Welcome to the Spring Integration JPA Sample! "
+ "\n "
+ "\n For more information please visit: "
+ "\n http://www.springintegration.org/ "
+ "\n "
+ "\n=========================================================");
System.out.println("Please enter a choice and press <enter>: ");
System.out.println("\t1. Use Hibernate");
@@ -70,21 +71,25 @@ public class Main {
System.out.println("\tq. Quit the application");
System.out.print("Enter you choice: ");
SpringApplicationBuilder springApplicationBuilder = new SpringApplicationBuilder(Main.class).web(false);
SpringApplicationBuilder springApplicationBuilder = new SpringApplicationBuilder(Main.class)
.web(WebApplicationType.NONE);
while (true) {
final String input = scanner.nextLine();
if("1".equals(input.trim())) {
if ("1".equals(input.trim())) {
springApplicationBuilder.sources(HibernateJpaAutoConfiguration.class);
break;
} else if("2".equals(input.trim())) {
}
else if ("2".equals(input.trim())) {
springApplicationBuilder.profiles("eclipseLink");
break;
} else if("q".equals(input.trim())) {
}
else if ("q".equals(input.trim())) {
System.out.println("Exiting application...bye.");
System.exit(0);
} else {
}
else {
System.out.println("Invalid choice\n\n");
System.out.print("Enter you choice: ");
}
@@ -129,12 +134,12 @@ public class Main {
}
private static void createPersonDetails(final Scanner scanner,PersonService service) {
while(true) {
private static void createPersonDetails(final Scanner scanner, PersonService service) {
while (true) {
System.out.print("\nEnter the Person's name:");
String name = null;
while(true) {
while (true) {
name = scanner.nextLine();
@@ -151,9 +156,9 @@ public class Main {
person = service.createPerson(person);
System.out.println("Created person record with id: " + person.getId());
System.out.print("Do you want to create another person? (y/n)");
String choice = scanner.nextLine();
String choice = scanner.nextLine();
if(!"y".equalsIgnoreCase(choice)) {
if (!"y".equalsIgnoreCase(choice)) {
break;
}
}
@@ -161,22 +166,23 @@ public class Main {
private static void findPeople(final PersonService service) {
System.out.println("ID NAME CREATED");
System.out.println("==================================");
System.out.println("ID NAME CREATED");
System.out.println("==================================");
final List<Person> people = service.findPeople();
final List<Person> people = service.findPeople();
if(people != null && !people.isEmpty()) {
for(Person person : people) {
System.out.print(String.format("%d, %s, ", person.getId(), person.getName()));
System.out.println(DATE_FORMAT.format(person.getCreatedDateTime()));//NOSONAR
}
} else {
System.out.println(
String.format("No Person record found."));
if (people != null && !people.isEmpty()) {
for (Person person : people) {
System.out.print(String.format("%d, %s, ", person.getId(), person.getName()));
System.out.println(DATE_FORMAT.format(person.getCreatedDateTime()));//NOSONAR
}
}
else {
System.out.println(
String.format("No Person record found."));
}
System.out.println("==================================\n\n");
System.out.println("==================================\n\n");
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2014-2015 the original author or authors.
* Copyright 2014-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,30 +16,28 @@
package org.springframework.integration.samples.sftp;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.security.KeyFactory;
import java.security.PublicKey;
import java.security.spec.RSAPublicKeySpec;
import java.util.Arrays;
import java.util.Collections;
import org.apache.sshd.SshServer;
import org.apache.sshd.common.NamedFactory;
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
import org.apache.sshd.common.util.Base64;
import org.apache.sshd.server.Command;
import org.apache.sshd.server.PublickeyAuthenticator;
import org.apache.sshd.server.SshServer;
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
import org.apache.sshd.server.session.ServerSession;
import org.apache.sshd.server.sftp.SftpSubsystem;
import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.SmartLifecycle;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.FileSystemResource;
import org.springframework.util.SocketUtils;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import org.springframework.util.Base64Utils;
import org.springframework.util.StreamUtils;
/**
@@ -47,7 +45,10 @@ import org.springframework.util.StreamUtils;
*/
public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle {
public static final int PORT = SocketUtils.findAvailableTcpPort();
/**
* Let OS to obtain the proper port
*/
public static final int PORT = 0;
private final SshServer server = SshServer.setUpDefaultServer();
@@ -55,31 +56,36 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle {
private volatile boolean running;
private DefaultSftpSessionFactory defaultSftpSessionFactory;
public void setPort(int port) {
this.port = port;
}
public void setDefaultSftpSessionFactory(DefaultSftpSessionFactory defaultSftpSessionFactory) {
this.defaultSftpSessionFactory = defaultSftpSessionFactory;
}
@Override
public void afterPropertiesSet() throws Exception {
final PublicKey allowedKey = decodePublicKey();
this.server.setPublickeyAuthenticator(new PublickeyAuthenticator() {
@Override
public boolean authenticate(String username, PublicKey key, ServerSession session) {
return key.equals(allowedKey);
}
});
this.server.setPublickeyAuthenticator((username, key, session) -> key.equals(allowedKey));
this.server.setPort(this.port);
this.server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider("hostkey.ser"));
this.server.setSubsystemFactories(Collections.<NamedFactory<Command>>singletonList(new SftpSubsystem.Factory()));
final String virtualDir = new FileSystemResource("").getFile().getAbsolutePath();
server.setFileSystemFactory(new VirtualFileSystemFactory(virtualDir));
this.server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(new File("hostkey.ser")));
this.server.setSubsystemFactories(Collections.singletonList(new SftpSubsystemFactory()));
final String pathname = System.getProperty("java.io.tmpdir") + File.separator + "sftptest" + File.separator;
new File(pathname).mkdirs();
server.setFileSystemFactory(new VirtualFileSystemFactory(Paths.get(pathname)));
}
private PublicKey decodePublicKey() throws Exception {
InputStream stream = new ClassPathResource("META-INF/keys/sftp_rsa.pub").getInputStream();
byte[] decodeBuffer = Base64.decodeBase64(StreamUtils.copyToByteArray(stream));
byte[] keyBytes = StreamUtils.copyToByteArray(stream);
// strip any newline chars
while (keyBytes[keyBytes.length - 1] == 0x0a || keyBytes[keyBytes.length - 1] == 0x0d) {
keyBytes = Arrays.copyOf(keyBytes, keyBytes.length - 1);
}
byte[] decodeBuffer = Base64Utils.decode(keyBytes);
ByteBuffer bb = ByteBuffer.wrap(decodeBuffer);
int len = bb.getInt();
byte[] type = new byte[len];
@@ -116,8 +122,9 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle {
@Override
public void start() {
try {
server.start();
this.running = true;
this.server.start();
this.defaultSftpSessionFactory.setPort(this.server.getPort());
this.running = true;
}
catch (IOException e) {
throw new IllegalStateException(e);
@@ -136,7 +143,7 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle {
try {
server.stop(true);
}
catch (InterruptedException e) {
catch (Exception e) {
throw new IllegalStateException(e);
}
finally {
@@ -149,5 +156,4 @@ public class EmbeddedSftpServer implements InitializingBean, SmartLifecycle {
public boolean isRunning() {
return this.running;
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2014 the original author or authors.
* Copyright 2014-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,19 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.samples.sftp;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.file.remote.SessionCallback;
import org.springframework.integration.file.remote.session.Session;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.ChannelSftp.LsEntry;
@@ -34,6 +32,8 @@ import com.jcraft.jsch.SftpException;
/**
* @author Gary Russell
* @author Artem Bilan
*
* @since 4.1
*
*/
@@ -42,78 +42,57 @@ public class SftpTestUtils {
public static void createTestFiles(RemoteFileTemplate<LsEntry> template, final String... fileNames) {
if (template != null) {
final ByteArrayInputStream stream = new ByteArrayInputStream("foo".getBytes());
template.execute(new SessionCallback<LsEntry, Void>() {
@Override
public Void doInSession(Session<LsEntry> session) throws IOException {
try {
session.mkdir("si.sftp.sample");
}
catch (Exception e) {
assertThat(e.getMessage(), containsString("failed to create"));
}
for (int i = 0; i < fileNames.length; i++) {
stream.reset();
session.write(stream, "si.sftp.sample/" + fileNames[i]);
}
return null;
template.execute((SessionCallback<LsEntry, Void>) session -> {
try {
session.mkdir("si.sftp.sample");
}
catch (Exception e) {
assertThat(e.getMessage(), containsString("failed to create"));
}
for (int i = 0; i < fileNames.length; i++) {
stream.reset();
session.write(stream, "si.sftp.sample/" + fileNames[i]);
}
return null;
});
}
}
public static void cleanUp(RemoteFileTemplate<LsEntry> template, final String... fileNames) {
if (template != null) {
template.execute(new SessionCallback<LsEntry, Void>() {
@Override
public Void doInSession(Session<LsEntry> session) throws IOException {
// TODO: avoid DFAs with Spring 4.1 (INT-3412)
ChannelSftp channel = (ChannelSftp) new DirectFieldAccessor(new DirectFieldAccessor(session)
.getPropertyValue("targetSession")).getPropertyValue("channel");
for (int i = 0; i < fileNames.length; i++) {
try {
session.remove("si.sftp.sample/" + fileNames[i]);
}
catch (IOException e) {}
}
template.execute((SessionCallback<LsEntry, Void>) session -> {
for (int i = 0; i < fileNames.length; i++) {
try {
// should be empty
channel.rmdir("si.sftp.sample");
session.remove("si.sftp.sample/" + fileNames[i]);
}
catch (SftpException e) {
fail("Expected remote directory to be empty " + e.getMessage());
}
return null;
catch (IOException e) {}
}
// should be empty
session.rmdir("si.sftp.sample");
return null;
});
}
}
public static boolean fileExists(RemoteFileTemplate<LsEntry> template, final String... fileNames) {
if (template != null) {
return template.execute(new SessionCallback<LsEntry, Boolean>() {
@Override
public Boolean doInSession(Session<LsEntry> session) throws IOException {
// TODO: avoid DFAs with Spring 4.1 (INT-3412)
ChannelSftp channel = (ChannelSftp) new DirectFieldAccessor(new DirectFieldAccessor(session)
.getPropertyValue("targetSession")).getPropertyValue("channel");
for (int i = 0; i < fileNames.length; i++) {
try {
SftpATTRS stat = channel.stat("si.sftp.sample/" + fileNames[i]);
if (stat == null) {
System.out.println("stat returned null for " + fileNames[i]);
return false;
}
}
catch (SftpException e) {
System.out.println("Remote file not present: " + e.getMessage() + ": " + fileNames[i]);
return template.execute(session -> {
ChannelSftp channel = (ChannelSftp) session.getClientInstance();
for (int i = 0; i < fileNames.length; i++) {
try {
SftpATTRS stat = channel.stat("si.sftp.sample/" + fileNames[i]);
if (stat == null) {
System.out.println("stat returned null for " + fileNames[i]);
return false;
}
}
return true;
catch (SftpException e) {
System.out.println("Remote file not present: " + e.getMessage() + ": " + fileNames[i]);
return false;
}
}
return true;
});
}
else {

View File

@@ -9,20 +9,6 @@
<import resource="SftpSampleCommon.xml"/>
<bean id="sftpSessionFactory" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg ref="defaultSftpSessionFactory" />
</bean>
<bean id="defaultSftpSessionFactory"
class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<property name="host" value="${host}"/>
<property name="privateKey" value="${private.keyfile}"/>
<property name="privateKeyPassphrase" value="${passphrase}"/>
<property name="port" value="#{serverPort}"/>
<property name="user" value="${username}"/>
<property name="allowUnknownKeys" value="true"/>
</bean>
<int-sftp:inbound-channel-adapter id="sftpInbondAdapter"
auto-startup="false"
channel="receiveChannel"

View File

@@ -12,20 +12,6 @@
<int:gateway id="gw" service-interface="org.springframework.integration.samples.sftp.ToSftpFlowGateway"
default-request-channel="inbound"/>
<bean id="sftpSessionFactory" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg ref="defaultSftpSessionFactory" />
</bean>
<bean id="defaultSftpSessionFactory"
class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<property name="host" value="${host}"/>
<property name="privateKey" value="${private.keyfile}"/>
<property name="privateKeyPassphrase" value="${passphrase}"/>
<property name="port" value="#{serverPort}"/>
<property name="user" value="${username}"/>
<property name="allowUnknownKeys" value="true"/>
</bean>
<int-sftp:outbound-gateway id="gatewayLS"
session-factory="sftpSessionFactory"
request-channel="inbound"

View File

@@ -9,20 +9,6 @@
<import resource="SftpSampleCommon.xml"/>
<bean id="sftpSessionFactory" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg ref="defaultSftpSessionFactory" />
</bean>
<bean id="defaultSftpSessionFactory"
class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<property name="host" value="${host}"/>
<property name="privateKey" value="${private.keyfile}"/>
<property name="privateKeyPassphrase" value="${passphrase}"/>
<property name="port" value="#{serverPort}"/>
<property name="user" value="${username}"/>
<property name="allowUnknownKeys" value="true"/>
</bean>
<int:channel id="inputChannel"/>
<int-sftp:outbound-channel-adapter id="sftpOutboundAdapter"

View File

@@ -9,12 +9,23 @@
<context:property-placeholder location="classpath:user.properties"/>
<bean id="serverPort" class="java.lang.String">
<constructor-arg value="#{'${port}' == '-1' ? T(org.springframework.integration.samples.sftp.EmbeddedSftpServer).PORT : '${port}'}"/>
<bean id="sftpSessionFactory" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg ref="defaultSftpSessionFactory" />
</bean>
<bean id="defaultSftpSessionFactory"
class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<property name="host" value="${sftp.host}"/>
<property name="privateKey" value="${sftp.private.keyfile}"/>
<property name="privateKeyPassphrase" value="${sftp.passphrase}"/>
<property name="port" value="${sftp.port}"/>
<property name="user" value="${sftp.username}"/>
<property name="allowUnknownKeys" value="true"/>
</bean>
<bean class="org.springframework.integration.samples.sftp.EmbeddedSftpServer">
<property name="port" value="#{serverPort}"/>
<property name="port" value="${sftp.port}"/>
<property name="defaultSftpSessionFactory" ref="defaultSftpSessionFactory"/>
</bean>
</beans>

View File

@@ -1,7 +1,7 @@
host=localhost
# -1 means the embedded Apache MINA SshServer. Change it to any real port, if you are going to test sample against real SFTP Server
port=-1
username=user
passphrase=password
sftp.host=localhost
# 0 means the embedded Apache MINA SshServer. Change it to any real port, if you are going to test sample against real SFTP Server
sftp.port=0
sftp.username=user
sftp.passphrase=password
#private.keyfile=file:/home/someuser/.ssh/id_rsa
private.keyfile=classpath:META-INF/keys/sftp_rsa
sftp.private.keyfile=classpath:META-INF/keys/sftp_rsa

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2014 the original author or authors.
* Copyright 2014-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -31,8 +31,8 @@ import java.util.concurrent.TimeUnit;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.context.embedded.LocalServerPort;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.web.server.LocalServerPort;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.channel.DirectChannel;

View File

@@ -157,7 +157,7 @@ subprojects { subproject ->
ext {
activeMqVersion = '5.13.4'
apacheSshdVersion = '0.14.0'
apacheSshdVersion = '1.4.0'
aspectjVersion = '1.8.9'
commonsDigesterVersion = '2.0'
commonsDbcpVersion = '1.2.2'
@@ -171,7 +171,7 @@ subprojects { subproject ->
hamcrestVersion = '1.3'
hibernateVersion = '5.0.9.Final'
hibernateValidatorVersion = '4.2.0.Final'
ftpServerVersion = '1.0.6'
ftpServerVersion = '1.1.0'
flexjsonVersion = '2.0'
guavaVersion = '16.0.1'
groovyVersion = '2.3.0'