Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-2652 Fix mget for FTP

Browse Source 
FTP servers return full path from listNames() whereas SFTP
returns just the filename. mget logic assumed SFTP behavior.

Add a test to see if the filename already starts with the remote
directory and remove it before calling get() (which assumes just
the filename).

Includes an @Ignored test for FTP and SFTP that runs an mget
against a server.
This commit is contained in:
Gary Russell
2012-07-05 16:55:33 -04:00
parent c5bd22d336
commit 26ee9ab953
7 changed files with 249 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
View File

@@ -47,7 +47,7 @@ import org.springframework.util.StringUtils;
/**
* Base class for Outbound Gateways that perform remote file operations.
*
*
* @author Gary Russell
* @since 2.1
*/
@@ -197,7 +197,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
@Override
protected Object handleRequestMessage(Message<?> requestMessage) {
Session<F> session = this.sessionFactory.getSession();
Session<F> session = this.sessionFactory.getSession();
try {
if (COMMAND_LS.equals(this.command)) {
String dir = this.processor.processMessage(requestMessage);
@@ -378,20 +378,32 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
protected List<File> mGet(Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
Assert.isInstanceOf(ExtendedSession.class, session, "mget failed - ");
String path = fixPath(remoteDirectory, remoteFilename);
String path = generateFullPath(remoteDirectory, remoteFilename);
String[] fileNames = ((ExtendedSession<F>) session).listNames(path);
if (fileNames == null) {
fileNames = new String[0];
}
if (fileNames.length == 0 && this.options.contains(OPTION_EXCEPTION_WHEN_EMPTY)) {
throw new MessagingException("No files found at " + remoteDirectory
+ " with pattern " + remoteFilename);
}
List<File> files = new ArrayList<File>();
for (String fileName : fileNames) {
files.add(this.get(session, fixPath(remoteDirectory, fileName), fileName, false));
File file;
if (fileName.contains(this.remoteFileSeparator) &&
fileName.startsWith(remoteDirectory)) { // the server returned the full path
file = this.get(session, fileName,
fileName.substring(fileName.lastIndexOf(this.remoteFileSeparator)), false);
}
else {
file = this.get(session, generateFullPath(remoteDirectory, fileName), fileName, false);
}
files.add(file);
}
return files;
}
private String fixPath(String remoteDirectory, String remoteFilename) {
private String generateFullPath(String remoteDirectory, String remoteFilename) {
String path;
if (this.remoteFileSeparator.equals(remoteDirectory)) {
path = remoteFilename;
@@ -413,7 +425,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
int index = remoteFilePath.lastIndexOf(this.remoteFileSeparator);
if (index < 0) {
remoteFileName = remoteFilePath;
}
}
else {
remoteFileName = remoteFilePath.substring(index + 1);
}

64
spring-integration-file/src/main/java/org/springframework/integration/file/test/BigMGetTests.java Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.test;
import java.io.File;
import java.io.FileOutputStream;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.Message;
import org.springframework.integration.MessageChannel;
import org.springframework.integration.core.PollableChannel;
import org.springframework.integration.message.GenericMessage;
/**
* Created this test because a customer reported hanging with large mget.
*
* Could not reproduce; code is checked in, but test is @Ignored.
* @author Gary Russell
* @since 2.2
*
*/
public abstract class BigMGetTests {
protected static final int FILES = 600;
@Autowired
private MessageChannel inbound;
@Autowired
private PollableChannel resultChannel;
protected Message<List<File>> mgetManyFiles() throws Exception {
byte[] buff = new byte[150];
for (int i = 0; i < FILES; i++) {
File f = new File("/tmp/bigmget/file" + i);
f.createNewFile();
FileOutputStream fos = new FileOutputStream(f);
fos.write(buff);
fos.close();
}
inbound.send(new GenericMessage<String>("/tmp/bigmget/f*"));
for (int i = 0; i < FILES; i++) {
new File("/tmp/bigmget/file" + i).delete();
new File("/tmp/out/file" + i).delete();
}
@SuppressWarnings("unchecked")
Message<List<File>> results = (Message<List<File>>) resultChannel.receive(600000);
return results;
}
}

34
spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java
View File

@@ -54,7 +54,7 @@ public class RemoteFileOutboundGatewayTests {
private String tmpDir = System.getProperty("java.io.tmpdir");
@Test(expected=IllegalArgumentException.class)
public void testBad() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
@@ -96,6 +96,19 @@ public class RemoteFileOutboundGatewayTests {
@Test
public void testMGetWild() throws Exception {
testMGetWildGuts("f1", "f2");
}
/**
* Test a wildcard mget where the full path is returned for each file
* @throws Exception
*/
@Test
public void testMGetWildFullPath() throws Exception {
testMGetWildGuts("testremote/f1", "testremote/f2");
}
private void testMGetWildGuts(final String path1, final String path2) {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "mget", "payload");
@@ -104,6 +117,7 @@ public class RemoteFileOutboundGatewayTests {
new File(this.tmpDir + "/f1").delete();
new File(this.tmpDir + "/f2").delete();
when(sessionFactory.getSession()).thenReturn(new ExtendedSession() {
int n;
public boolean remove(String path) throws IOException {
return false;
}
@@ -112,6 +126,12 @@ public class RemoteFileOutboundGatewayTests {
}
public void read(String source, OutputStream outputStream)
throws IOException {
if (n++ == 0) {
assertEquals("testremote/f1", source);
}
else {
assertEquals("testremote/f2", source);
}
outputStream.write("testData".getBytes());
}
public void write(InputStream inputStream, String destination)
@@ -132,7 +152,7 @@ public class RemoteFileOutboundGatewayTests {
return false;
}
public String[] listNames(String path) throws IOException {
return new String[] {"f1", "f2"};
return new String[] {path1, path2};
}
});
@SuppressWarnings("unchecked")
@@ -416,7 +436,7 @@ public class RemoteFileOutboundGatewayTests {
public boolean remove(String path) throws IOException {
return false;
}
public TestLsEntry[] list(String path) throws IOException {
return new TestLsEntry[] {
new TestLsEntry("f1", 1234, false, false, 12345, "-rw-r--r--")
@@ -475,7 +495,7 @@ public class RemoteFileOutboundGatewayTests {
public boolean remove(String path) throws IOException {
return false;
}
public TestLsEntry[] list(String path) throws IOException {
return new TestLsEntry[] {
new TestLsEntry("f1", 1234, false, false, modified.getTime(), "-rw-r--r--")
@@ -499,7 +519,7 @@ public class RemoteFileOutboundGatewayTests {
}
public boolean isOpen() {
return open;
}
}
public boolean exists(String path) throws IOException {
return true;
}
@@ -531,7 +551,7 @@ public class RemoteFileOutboundGatewayTests {
public boolean remove(String path) throws IOException {
return false;
}
public TestLsEntry[] list(String path) throws IOException {
return new TestLsEntry[] {
new TestLsEntry("f1", 1234, false, false, 12345, "-rw-r--r--")
@@ -555,7 +575,7 @@ public class RemoteFileOutboundGatewayTests {
}
public boolean isOpen() {
return open;
}
}
public boolean exists(String path) throws IOException {
return true;
}

31
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/BigMGetTests-context.xml Normal file
View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-ftp="http://www.springframework.org/schema/integration/ftp"
xmlns:int="http://www.springframework.org/schema/integration"
xsi:schemaLocation="http://www.springframework.org/schema/integration/ftp http://www.springframework.org/schema/integration/ftp/spring-integration-ftp.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="ftpSessionFactory"
class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="localhost"/>
<property name="username" value="ftptest"/>
<property name="password" value="ftptest"/>
</bean>
<int:channel id="inbound" />
<int-ftp:outbound-gateway id="gatewayLS" cache-sessions="false"
session-factory="ftpSessionFactory"
request-channel="inbound"
command="mget"
command-options=""
expression="payload"
local-directory="/tmp/out"
reply-channel="resultChannel"/>
<int:channel id="resultChannel">
<int:queue />
</int:channel>
</beans>

39
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/BigMGetTests.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.ftp.outbound;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Gary Russell
* @since 2.2
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class BigMGetTests extends org.springframework.integration.file.test.BigMGetTests {
@Test @Ignore // needs directories and server (FTP and SFTP)
public void doTest() throws Exception {
assertEquals(FILES, this.mgetManyFiles().getPayload().size());
}
}

31
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/BigMGetTests-context.xml Normal file
View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-sftp="http://www.springframework.org/schema/integration/sftp"
xsi:schemaLocation="http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/sftp http://www.springframework.org/schema/integration/sftp/spring-integration-sftp-2.2.xsd">
<bean id="ftpSessionFactory"
class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<property name="host" value="localhost"/>
<property name="user" value="ftptest"/>
<property name="password" value="ftptest"/>
</bean>
<int:channel id="inbound" />
<int-sftp:outbound-gateway id="gatewayLS" cache-sessions="false"
session-factory="ftpSessionFactory"
request-channel="inbound"
command="mget"
command-options=""
expression="payload"
local-directory="/tmp/out"
reply-channel="resultChannel"/>
<int:channel id="resultChannel">
<int:queue />
</int:channel>
</beans>

39
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/BigMGetTests.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.outbound;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Gary Russell
* @since 2.2
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class BigMGetTests extends org.springframework.integration.file.test.BigMGetTests {
@Test @Ignore // needs directories and server (FTP and SFTP)
public void doTest() throws Exception {
assertEquals(FILES, this.mgetManyFiles().getPayload().size());
}
}