Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #537 from johndoe/INT-2652

Browse Source 
* INT-2652:
  INT-2652 Fix mget for FTP
This commit is contained in:
Oleg Zhurakousky
2012-07-10 17:07:32 -04:00
parent e73d3a3425 b8f7d01f69
commit d0f26cd618
7 changed files with 243 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
View File

@@ -46,7 +46,7 @@ import org.springframework.util.StringUtils;
/**
* Base class for Outbound Gateways that perform remote file operations.
*
*
* @author Gary Russell
* @since 2.1
*/
@@ -196,7 +196,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
@Override
protected Object handleRequestMessage(Message<?> requestMessage) {
Session<F> session = this.sessionFactory.getSession();
Session<F> session = this.sessionFactory.getSession();
try {
if (COMMAND_LS.equals(this.command)) {
String dir = this.processor.processMessage(requestMessage);
@@ -376,7 +376,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
protected List<File> mGet(Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
String path = fixPath(remoteDirectory, remoteFilename);
String path = generateFullPath(remoteDirectory, remoteFilename);
String[] fileNames = session.listNames(path);
if (fileNames == null) {
fileNames = new String[0];
@@ -387,12 +387,21 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
}
List<File> files = new ArrayList<File>();
for (String fileName : fileNames) {
files.add(this.get(session, fixPath(remoteDirectory, fileName), fileName, false));
File file;
if (fileName.contains(this.remoteFileSeparator) &&
fileName.startsWith(remoteDirectory)) { // the server returned the full path
file = this.get(session, fileName,
fileName.substring(fileName.lastIndexOf(this.remoteFileSeparator)), false);
}
else {
file = this.get(session, generateFullPath(remoteDirectory, fileName), fileName, false);
}
files.add(file);
}
return files;
}
private String fixPath(String remoteDirectory, String remoteFilename) {
private String generateFullPath(String remoteDirectory, String remoteFilename) {
String path;
if (this.remoteFileSeparator.equals(remoteDirectory)) {
path = remoteFilename;
@@ -414,7 +423,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
int index = remoteFilePath.lastIndexOf(this.remoteFileSeparator);
if (index < 0) {
remoteFileName = remoteFilePath;
}
}
else {
remoteFileName = remoteFilePath.substring(index + 1);
}

64
spring-integration-file/src/main/java/org/springframework/integration/file/test/BigMGetTests.java Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.test;
import java.io.File;
import java.io.FileOutputStream;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.Message;
import org.springframework.integration.MessageChannel;
import org.springframework.integration.core.PollableChannel;
import org.springframework.integration.message.GenericMessage;
/**
* Created this test because a customer reported hanging with large mget.
*
* Could not reproduce; code is checked in, but test is @Ignored.
* @author Gary Russell
* @since 2.2
*
*/
public abstract class BigMGetTests {
protected static final int FILES = 600;
@Autowired
private MessageChannel inbound;
@Autowired
private PollableChannel resultChannel;
protected Message<List<File>> mgetManyFiles() throws Exception {
byte[] buff = new byte[150];
for (int i = 0; i < FILES; i++) {
File f = new File("/tmp/bigmget/file" + i);
f.createNewFile();
FileOutputStream fos = new FileOutputStream(f);
fos.write(buff);
fos.close();
}
inbound.send(new GenericMessage<String>("/tmp/bigmget/f*"));
for (int i = 0; i < FILES; i++) {
new File("/tmp/bigmget/file" + i).delete();
new File("/tmp/out/file" + i).delete();
}
@SuppressWarnings("unchecked")
Message<List<File>> results = (Message<List<File>>) resultChannel.receive(600000);
return results;
}
}

29
spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java
View File

@@ -38,7 +38,6 @@ import org.springframework.integration.Message;
import org.springframework.integration.MessagingException;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.filters.AbstractSimplePatternFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
@@ -55,7 +54,7 @@ public class RemoteFileOutboundGatewayTests {
private String tmpDir = System.getProperty("java.io.tmpdir");
@Test(expected=IllegalArgumentException.class)
public void testBad() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
@@ -130,6 +129,19 @@ public class RemoteFileOutboundGatewayTests {
@Test
public void testMGetWild() throws Exception {
testMGetWildGuts("f1", "f2");
}
/**
* Test a wildcard mget where the full path is returned for each file
* @throws Exception
*/
@Test
public void testMGetWildFullPath() throws Exception {
testMGetWildGuts("testremote/f1", "testremote/f2");
}
private void testMGetWildGuts(final String path1, final String path2) {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "mget", "payload");
@@ -138,6 +150,7 @@ public class RemoteFileOutboundGatewayTests {
new File(this.tmpDir + "/f1").delete();
new File(this.tmpDir + "/f2").delete();
when(sessionFactory.getSession()).thenReturn(new Session() {
int n;
public boolean remove(String path) throws IOException {
return false;
}
@@ -146,6 +159,12 @@ public class RemoteFileOutboundGatewayTests {
}
public void read(String source, OutputStream outputStream)
throws IOException {
if (n++ == 0) {
assertEquals("testremote/f1", source);
}
else {
assertEquals("testremote/f2", source);
}
outputStream.write("testData".getBytes());
}
public void write(InputStream inputStream, String destination)
@@ -166,7 +185,7 @@ public class RemoteFileOutboundGatewayTests {
return false;
}
public String[] listNames(String path) throws IOException {
return new String[] {"f1", "f2"};
return new String[] {path1, path2};
}
});
@SuppressWarnings("unchecked")
@@ -533,7 +552,7 @@ public class RemoteFileOutboundGatewayTests {
}
public boolean isOpen() {
return open;
}
}
public boolean exists(String path) throws IOException {
return true;
}
@@ -591,7 +610,7 @@ public class RemoteFileOutboundGatewayTests {
}
public boolean isOpen() {
return open;
}
}
public boolean exists(String path) throws IOException {
return true;
}

31
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/BigMGetTests-context.xml Normal file
View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-ftp="http://www.springframework.org/schema/integration/ftp"
xmlns:int="http://www.springframework.org/schema/integration"
xsi:schemaLocation="http://www.springframework.org/schema/integration/ftp http://www.springframework.org/schema/integration/ftp/spring-integration-ftp.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="ftpSessionFactory"
class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="localhost"/>
<property name="username" value="ftptest"/>
<property name="password" value="ftptest"/>
</bean>
<int:channel id="inbound" />
<int-ftp:outbound-gateway id="gatewayLS" cache-sessions="false"
session-factory="ftpSessionFactory"
request-channel="inbound"
command="mget"
command-options=""
expression="payload"
local-directory="/tmp/out"
reply-channel="resultChannel"/>
<int:channel id="resultChannel">
<int:queue />
</int:channel>
</beans>

39
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/BigMGetTests.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.ftp.outbound;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Gary Russell
* @since 2.2
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class BigMGetTests extends org.springframework.integration.file.test.BigMGetTests {
@Test @Ignore // needs directories and server (FTP and SFTP)
public void doTest() throws Exception {
assertEquals(FILES, this.mgetManyFiles().getPayload().size());
}
}

31
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/BigMGetTests-context.xml Normal file
View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-sftp="http://www.springframework.org/schema/integration/sftp"
xsi:schemaLocation="http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/sftp http://www.springframework.org/schema/integration/sftp/spring-integration-sftp-2.2.xsd">
<bean id="ftpSessionFactory"
class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
<property name="host" value="localhost"/>
<property name="user" value="ftptest"/>
<property name="password" value="ftptest"/>
</bean>
<int:channel id="inbound" />
<int-sftp:outbound-gateway id="gatewayLS" cache-sessions="false"
session-factory="ftpSessionFactory"
request-channel="inbound"
command="mget"
command-options=""
expression="payload"
local-directory="/tmp/out"
reply-channel="resultChannel"/>
<int:channel id="resultChannel">
<int:queue />
</int:channel>
</beans>

39
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/BigMGetTests.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.outbound;
import static org.junit.Assert.assertEquals;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Gary Russell
* @since 2.2
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class BigMGetTests extends org.springframework.integration.file.test.BigMGetTests {
@Test @Ignore // needs directories and server (FTP and SFTP)
public void doTest() throws Exception {
assertEquals(FILES, this.mgetManyFiles().getPayload().size());
}
}