diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java index 93269f7a59..3c4a367c78 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ import org.springframework.util.StringUtils; * @author Oleg Zhurakousky * @author Mark Fisher * @author Gary Russell + * @author Artem Bilan * @since 2.0 */ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser { @@ -46,8 +47,10 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst // configure the InboundFileSynchronizer properties BeanDefinition expressionDef = IntegrationNamespaceUtils.createExpressionDefinitionFromValueOrExpression( - "remote-directory", "remote-directory-expression", parserContext, element, true); - synchronizerBuilder.addPropertyValue("remoteDirectoryExpression", expressionDef); + "remote-directory", "remote-directory-expression", parserContext, element, false); + if (expressionDef != null) { + synchronizerBuilder.addPropertyValue("remoteDirectoryExpression", expressionDef); + } IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "delete-remote-files"); IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "preserve-timestamp"); @@ -57,7 +60,8 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst this.configureFilter(synchronizerBuilder, element, parserContext); // build the MessageSource - BeanDefinitionBuilder messageSourceBuilder = BeanDefinitionBuilder.genericBeanDefinition(this.getMessageSourceClassname()); + BeanDefinitionBuilder messageSourceBuilder = + BeanDefinitionBuilder.genericBeanDefinition(getMessageSourceClassname()); messageSourceBuilder.addConstructorArgValue(synchronizerBuilder.getBeanDefinition()); String comparator = element.getAttribute("comparator"); if (StringUtils.hasText(comparator)) { @@ -65,17 +69,21 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst } IntegrationNamespaceUtils.setReferenceIfAttributeDefined(messageSourceBuilder, element, "local-filter"); IntegrationNamespaceUtils.setValueIfAttributeDefined(messageSourceBuilder, element, "local-directory"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(messageSourceBuilder, element, "auto-create-local-directory"); + IntegrationNamespaceUtils.setValueIfAttributeDefined(messageSourceBuilder, element, + "auto-create-local-directory"); String localFileGeneratorExpression = element.getAttribute("local-filename-generator-expression"); if (StringUtils.hasText(localFileGeneratorExpression)) { - BeanDefinitionBuilder localFileGeneratorExpressionBuilder = BeanDefinitionBuilder.genericBeanDefinition(ExpressionFactoryBean.class); + BeanDefinitionBuilder localFileGeneratorExpressionBuilder = + BeanDefinitionBuilder.genericBeanDefinition(ExpressionFactoryBean.class); localFileGeneratorExpressionBuilder.addConstructorArgValue(localFileGeneratorExpression); - synchronizerBuilder.addPropertyValue("localFilenameGeneratorExpression", localFileGeneratorExpressionBuilder.getBeanDefinition()); + synchronizerBuilder.addPropertyValue("localFilenameGeneratorExpression", + localFileGeneratorExpressionBuilder.getBeanDefinition()); } return messageSourceBuilder.getBeanDefinition(); } - private void configureFilter(BeanDefinitionBuilder synchronizerBuilder, Element element, ParserContext parserContext) { + private void configureFilter(BeanDefinitionBuilder synchronizerBuilder, Element element, + ParserContext parserContext) { String filter = element.getAttribute("filter"); String fileNamePattern = element.getAttribute("filename-pattern"); String fileNameRegex = element.getAttribute("filename-regex"); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java index 35051df056..83bda34083 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,7 +58,9 @@ public abstract class AbstractRemoteFileOutboundGatewayParser extends AbstractCo } else { builder.addConstructorArgValue(element.getAttribute("command")); - builder.addConstructorArgValue(element.getAttribute(EXPRESSION_ATTRIBUTE)); + if (element.hasAttribute(EXPRESSION_ATTRIBUTE)) { + builder.addConstructorArgValue(element.getAttribute(EXPRESSION_ATTRIBUTE)); + } } IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "command-options", "options"); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "reply-timeout", "sendTimeout"); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index 6df184bc87..29d5761c13 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,7 +47,6 @@ import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.file.support.FileExistsMode; import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor; -import org.springframework.integration.support.AbstractIntegrationMessageBuilder; import org.springframework.integration.support.PartialSuccessException; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHandlingException; @@ -65,7 +64,7 @@ import org.springframework.util.StringUtils; */ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReplyProducingMessageHandler { - private final RemoteFileTemplate remoteFileTemplate; + protected final RemoteFileTemplate remoteFileTemplate; protected final Command command; @@ -512,7 +511,7 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply private Object doLs(Message requestMessage) { String dir = this.fileNameProcessor.processMessage(requestMessage); - if (!dir.endsWith(this.remoteFileTemplate.getRemoteFileSeparator())) { + if (dir != null && !dir.endsWith(this.remoteFileTemplate.getRemoteFileSeparator())) { dir += this.remoteFileTemplate.getRemoteFileSeparator(); } final String fullDir = dir; @@ -529,9 +528,9 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply } private Object doGet(final Message requestMessage) { - final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); - final String remoteFilename = this.getRemoteFilename(remoteFilePath); - final String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); + final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); + final String remoteFilename = getRemoteFilename(remoteFilePath); + final String remoteDir = getRemoteDirectory(remoteFilePath, remoteFilename); Session session = null; Object payload; if (this.options.contains(Option.STREAM)) { @@ -550,30 +549,27 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply @Override public File doInSession(Session session) throws IOException { - return AbstractRemoteFileOutboundGateway.this.get(requestMessage, session, remoteDir, remoteFilePath, - remoteFilename, true); + return get(requestMessage, session, remoteDir, remoteFilePath, remoteFilename, true); } }); } - AbstractIntegrationMessageBuilder builder = this.getMessageBuilderFactory().withPayload(payload) - .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) - .setHeader(FileHeaders.REMOTE_FILE, remoteFilename); - if (session != null) { - builder.setHeader(FileHeaders.REMOTE_SESSION, session); - } - return builder.build(); + return getMessageBuilderFactory().withPayload(payload) + .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) + .setHeader(FileHeaders.REMOTE_FILE, remoteFilename) + .setHeader(FileHeaders.REMOTE_SESSION, session) + .build(); } private Object doMget(final Message requestMessage) { - final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); - final String remoteFilename = this.getRemoteFilename(remoteFilePath); - final String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); + final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); + final String remoteFilename = getRemoteFilename(remoteFilePath); + final String remoteDir = getRemoteDirectory(remoteFilePath, remoteFilename); List payload = this.remoteFileTemplate.execute(new SessionCallback>() { @Override public List doInSession(Session session) throws IOException { - return AbstractRemoteFileOutboundGateway.this.mGet(requestMessage, session, remoteDir, remoteFilename); + return mGet(requestMessage, session, remoteDir, remoteFilename); } }); return this.getMessageBuilderFactory().withPayload(payload) @@ -583,10 +579,12 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply } private Object doRm(Message requestMessage) { - final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); - String remoteFilename = this.getRemoteFilename(remoteFilePath); - String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); + final String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); + String remoteFilename = getRemoteFilename(remoteFilePath); + String remoteDir = getRemoteDirectory(remoteFilePath, remoteFilename); + boolean payload = this.remoteFileTemplate.remove(remoteFilePath); + return this.getMessageBuilderFactory().withPayload(payload) .setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir) .setHeader(FileHeaders.REMOTE_FILE, remoteFilename) @@ -595,8 +593,8 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply private Object doMv(Message requestMessage) { String remoteFilePath = this.fileNameProcessor.processMessage(requestMessage); - String remoteFilename = this.getRemoteFilename(remoteFilePath); - String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename); + String remoteFilename = getRemoteFilename(remoteFilePath); + String remoteDir = getRemoteDirectory(remoteFilePath, remoteFilename); String remoteFileNewPath = this.renameProcessor.processMessage(requestMessage); Assert.hasLength(remoteFileNewPath, "New filename cannot be empty"); @@ -635,8 +633,7 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply return this.doPut(requestMessage); } else { - List replies = this.putLocalDirectory(requestMessage, file, null); - return replies; + return putLocalDirectory(requestMessage, file, null); } } @@ -717,7 +714,9 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply private List listFilesInRemoteDir(Session session, String directory, String subDirectory) throws IOException { List lsFiles = new ArrayList(); - F[] files = session.list(directory + subDirectory); + String remoteDirectory = buildRemotePath(directory, subDirectory); + + F[] files = session.list(remoteDirectory); boolean recursion = this.options.contains(Option.RECURSIVE); if (!ObjectUtils.isEmpty(files)) { Collection filteredFiles = this.filterFiles(files); @@ -742,6 +741,17 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply return lsFiles; } + private String buildRemotePath(String parent, String child) { + String remotePath = null; + if (parent != null) { + remotePath = (parent + child); + } + else if (StringUtils.hasText(child)) { + remotePath = "." + this.remoteFileTemplate.getRemoteFileSeparator() + child; + } + return remotePath; + } + protected final List filterFiles(F[] files) { return (this.filter != null) ? this.filter.filterFiles(files) : Arrays.asList(files); } @@ -784,8 +794,8 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply * @return The file. * @throws IOException Any IOException. */ - protected File get(Message message, Session session, String remoteDir, String remoteFilePath, String remoteFilename, boolean lsFirst) - throws IOException { + protected File get(Message message, Session session, String remoteDir, String remoteFilePath, + String remoteFilename, boolean lsFirst) throws IOException { F[] files = null; if (lsFirst) { files = session.list(remoteFilePath); @@ -796,7 +806,8 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply throw new MessagingException(remoteFilePath + " is not a file"); } } - File localFile = new File(this.generateLocalDirectory(message, remoteDir), this.generateLocalFileName(message, remoteFilename)); + File localFile = + new File(generateLocalDirectory(message, remoteDir), generateLocalFileName(message, remoteFilename)); FileExistsMode fileExistsMode = this.fileExistsMode; boolean appending = FileExistsMode.APPEND.equals(fileExistsMode); boolean replacing = FileExistsMode.REPLACE.equals(fileExistsMode); @@ -874,37 +885,40 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply private List mGetWithoutRecursion(Message message, Session session, String remoteDirectory, String remoteFilename) throws IOException { - String path = this.generateFullPath(remoteDirectory, remoteFilename); - String[] fileNames = session.listNames(path); - if (fileNames == null) { - fileNames = new String[0]; - } - if (fileNames.length == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) { - throw new MessagingException("No files found at " + remoteDirectory + List files = new ArrayList(); + String remotePath = buildRemotePath(remoteDirectory, remoteFilename); + @SuppressWarnings("unchecked") + List> remoteFiles = (List>) ls(session, remotePath); + if (remoteFiles.size() == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) { + throw new MessagingException("No files found at " + + (remoteDirectory != null ? remoteDirectory : "Client Working Directory") + " with pattern " + remoteFilename); } - List files = new ArrayList(); - String remoteFileSeparator = this.remoteFileTemplate.getRemoteFileSeparator(); try { - for (String fileName : fileNames) { - File file; - if (fileName.contains(remoteFileSeparator) && - fileName.startsWith(remoteDirectory)) { // the server returned the full path - file = this.get(message, session, remoteDirectory, fileName, - fileName.substring(fileName.lastIndexOf(remoteFileSeparator)), false); - } - else { - file = this.get(message, session, remoteDirectory, - this.generateFullPath(remoteDirectory, fileName), fileName, false); + for (AbstractFileInfo lsEntry : remoteFiles) { + if (lsEntry.isDirectory()) { + continue; } + String fullFileName = remoteDirectory != null + ? remoteDirectory + getFilename(lsEntry) + : getFilename(lsEntry); + /* + * With recursion, the filename might contain subdirectory information + * normalize each file separately. + */ + String fileName = this.getRemoteFilename(fullFileName); + String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName); + File file = this.get(message, session, actualRemoteDirectory, + fullFileName, fileName, false); files.add(file); } } catch (Exception e) { if (files.size() > 0) { throw new PartialSuccessException(message, - "Partially successful 'mget' operation on " + remoteDirectory, e, files, - Arrays.asList(fileNames)); + "Partially successful recursive 'mget' operation on " + + (remoteDirectory != null ? remoteDirectory : "Client Working Directory"), + e, files, remoteFiles); } else if (e instanceof MessagingException) { throw (MessagingException) e; @@ -920,14 +934,17 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply String remoteFilename) throws IOException { List files = new ArrayList(); @SuppressWarnings("unchecked") - List> fileNames = (List>) this.ls(session, remoteDirectory); + List> fileNames = (List>) ls(session, remoteDirectory); if (fileNames.size() == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) { - throw new MessagingException("No files found at " + remoteDirectory + throw new MessagingException("No files found at " + + (remoteDirectory != null ? remoteDirectory : "Client Working Directory") + " with pattern " + remoteFilename); } try { for (AbstractFileInfo lsEntry : fileNames) { - String fullFileName = remoteDirectory + this.getFilename(lsEntry); + String fullFileName = remoteDirectory != null + ? remoteDirectory + getFilename(lsEntry) + : getFilename(lsEntry); /* * With recursion, the filename might contain subdirectory information * normalize each file separately. @@ -942,7 +959,9 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply catch (Exception e) { if (files.size() > 0) { throw new PartialSuccessException(message, - "Partially successful recursive 'mget' operation on " + remoteDirectory, e, files, fileNames); + "Partially successful recursive 'mget' operation on " + + (remoteDirectory != null ? remoteDirectory : "Client Working Directory"), + e, files, fileNames); } else if (e instanceof MessagingException) { throw (MessagingException) e; @@ -957,45 +976,30 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply private String getRemoteDirectory(String remoteFilePath, String remoteFilename) { String remoteDir = remoteFilePath.substring(0, remoteFilePath.lastIndexOf(remoteFilename)); if (remoteDir.length() == 0) { - remoteDir = this.remoteFileTemplate.getRemoteFileSeparator(); + return null; } return remoteDir; } - private String generateFullPath(String remoteDirectory, String remoteFilename) { - String path; - String remoteFileSeparator = this.remoteFileTemplate.getRemoteFileSeparator(); - if (remoteFileSeparator.equals(remoteDirectory)) { - path = remoteFilename; - } - else if (remoteDirectory.endsWith(remoteFileSeparator)) { - path = remoteDirectory + remoteFilename; - } - else { - path = remoteDirectory + remoteFileSeparator + remoteFilename; - } - return path; - } - /** * @param remoteFilePath The remote file path. * @return The remote file name. */ protected String getRemoteFilename(String remoteFilePath) { - String remoteFileName; int index = remoteFilePath.lastIndexOf(this.remoteFileTemplate.getRemoteFileSeparator()); if (index < 0) { - remoteFileName = remoteFilePath; + return remoteFilePath; } else { - remoteFileName = remoteFilePath.substring(index + 1); + return remoteFilePath.substring(index + 1); } - return remoteFileName; } private File generateLocalDirectory(Message message, String remoteDirectory) { - EvaluationContext evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.getBeanFactory()); - evaluationContext.setVariable("remoteDirectory", remoteDirectory); + EvaluationContext evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory()); + if (remoteDirectory != null) { + evaluationContext.setVariable("remoteDirectory", remoteDirectory); + } //TODO see org.springframework.integration.context.CustomConversionServiceFactoryBean // File localDir = this.localDirectoryExpression.getValue(evaluationContext, message, File.class); String localDirPath = this.localDirectoryExpression.getValue(evaluationContext, message, String.class); @@ -1008,7 +1012,7 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply private String generateLocalFileName(Message message, String remoteFileName){ if (this.localFilenameGeneratorExpression != null){ - EvaluationContext evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.getBeanFactory()); + EvaluationContext evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory()); evaluationContext.setVariable("remoteFileName", remoteFileName); return this.localFilenameGeneratorExpression.getValue(evaluationContext, message, String.class); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java index c9e077791d..68899c66ef 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -200,6 +200,14 @@ public abstract class AbstractInboundFileSynchronizer if (this.evaluationContext == null) { this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.beanFactory); } + doInit(); + } + + /** + * Subclasses can override to perform initialization - called from + * {@link InitializingBean#afterPropertiesSet()}. + */ + protected void doInit() { } protected final List filterFiles(F[] files) { @@ -271,7 +279,9 @@ public abstract class AbstractInboundFileSynchronizer Session session) throws IOException { String remoteFileName = this.getFilename(remoteFile); String localFileName = this.generateLocalFileName(remoteFileName); - String remoteFilePath = remoteDirectoryPath + remoteFileSeparator + remoteFileName; + String remoteFilePath = remoteDirectoryPath != null + ? (remoteDirectoryPath + remoteFileSeparator + remoteFileName) + : remoteFileName; if (!this.isFile(remoteFile)) { if (logger.isDebugEnabled()) { logger.debug("cannot copy, not a file: " + remoteFilePath); diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java index d4f6964bcb..8e42a25c1a 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -188,8 +189,10 @@ public class RemoteFileOutboundGatewayTests { } @Override - public String[] listNames(String path) throws IOException { - return new String[] { path1, path2 }; + public TestLsEntry[] list(String path) throws IOException { + return new TestLsEntry[] { + new TestLsEntry(path1.replaceFirst("testremote/", ""), 123, false, false, 1234, "-r--r--r--"), + new TestLsEntry(path2.replaceFirst("testremote/", ""), 123, false, false, 1234, "-r--r--r--")}; } }); @@ -220,8 +223,8 @@ public class RemoteFileOutboundGatewayTests { } @Override - public String[] listNames(String path) throws IOException { - return new String[]{"f1"}; + public TestLsEntry[] list(String path) throws IOException { + return new TestLsEntry[]{new TestLsEntry("f1", 123, false, false, 1234, "-r--r--r--")}; } }); @@ -621,10 +624,8 @@ public class RemoteFileOutboundGatewayTests { assertEquals(outFile, out.getPayload()); assertTrue(outFile.exists()); outFile.delete(); - assertEquals("/", - out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); - assertEquals("f1", - out.getHeaders().get(FileHeaders.REMOTE_FILE)); + assertNull(out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); + assertEquals("f1", out.getHeaders().get(FileHeaders.REMOTE_FILE)); } @SuppressWarnings("unchecked") diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java index 008867ac2f..7862967096 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -82,6 +82,32 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway The {@code remoteDirectory} expression is {@code null} assuming to use + * the {@code workingDirectory} from the FTP Client. + * @param sessionFactory the session factory. + * @param command the command. + * @since 4.3 + */ + public FtpOutboundGateway(SessionFactory sessionFactory, String command) { + this(sessionFactory, command, null); + } + + /** + * Construct an instance with the supplied remote file template, a command ('ls', + * 'get' etc). + *

The {@code remoteDirectory} expression is {@code null} assuming to use + * the {@code workingDirectory} from the FTP Client. + * @param remoteFileTemplate the remote file template. + * @param command the command. + * @since 4.3 + */ + public FtpOutboundGateway(RemoteFileTemplate remoteFileTemplate, String command) { + this(remoteFileTemplate, command, null); + } + @Override public String getComponentType() { return "ftp:outbound-gateway"; diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpInboundFileSynchronizer.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpInboundFileSynchronizer.java index fcbcc95f70..0df12793ae 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpInboundFileSynchronizer.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpInboundFileSynchronizer.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package org.springframework.integration.ftp.inbound; import org.apache.commons.net.ftp.FTPFile; +import org.springframework.expression.common.LiteralExpression; import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer; @@ -40,9 +41,9 @@ public class FtpInboundFileSynchronizer extends AbstractInboundFileSynchronizer< */ public FtpInboundFileSynchronizer(SessionFactory sessionFactory) { super(sessionFactory); + setRemoteDirectoryExpression(new LiteralExpression(null)); } - @Override protected boolean isFile(FTPFile file) { return file != null && file.isFile(); diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/AbstractFtpSessionFactory.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/AbstractFtpSessionFactory.java index e1765764b2..86b5a6c7df 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/AbstractFtpSessionFactory.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/AbstractFtpSessionFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package org.springframework.integration.ftp.session; import java.io.IOException; -import java.net.SocketException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,8 +39,6 @@ import org.springframework.util.Assert; */ public abstract class AbstractFtpSessionFactory implements SessionFactory { - public static final String DEFAULT_REMOTE_WORKING_DIRECTORY = "/"; - private final Log logger = LogFactory.getLog(this.getClass()); protected FTPClientConfig config; @@ -172,7 +169,7 @@ public abstract class AbstractFtpSessionFactory implements } } - private T createClient() throws SocketException, IOException { + private T createClient() throws IOException { final T client = this.createClientInstance(); Assert.notNull(client, "client must not be null"); client.configure(this.config); @@ -200,7 +197,7 @@ public abstract class AbstractFtpSessionFactory implements // Login if (!client.login(username, password)) { - throw new IllegalStateException("Login failed. The respponse from the server is: " + + throw new IllegalStateException("Login failed. The response from the server is: " + client.getReplyString()); } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java index 3b9ffc51a8..da65f28408 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2013 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,6 +36,7 @@ import org.springframework.util.Assert; * @author Mark Fisher * @author Oleg Zhurakousky * @author Gary Russell + * @author Artem Bilan * @since 2.0 */ public class FtpSession implements Session { @@ -55,22 +56,21 @@ public class FtpSession implements Session { @Override public boolean remove(String path) throws IOException { Assert.hasText(path, "path must not be null"); - boolean completed = this.client.deleteFile(path); - if (!completed) { + if (!this.client.deleteFile(path)) { throw new IOException("Failed to delete '" + path + "'. Server replied with: " + client.getReplyString()); } - return completed; + else { + return true; + } } @Override public FTPFile[] list(String path) throws IOException { - Assert.hasText(path, "path must not be null"); return this.client.listFiles(path); } @Override public String[] listNames(String path) throws IOException { - Assert.hasText(path, "path must not be null"); return this.client.listNames(path); } @@ -83,7 +83,7 @@ public class FtpSession implements Session { throw new IOException("Failed to copy '" + path + "'. Server replied with: " + this.client.getReplyString()); } - logger.info("File has been successfully transfered from: " + path); + logger.info("File has been successfully transferred from: " + path); } @Override @@ -93,7 +93,8 @@ public class FtpSession implements Session { } InputStream inputStream = this.client.retrieveFileStream(source); if (inputStream == null) { - throw new IOException("Failed to obtain InputStream for remote file " + source + ": " + this.client.getReplyCode()); + throw new IOException("Failed to obtain InputStream for remote file " + source + ": " + + this.client.getReplyCode()); } return inputStream; } @@ -123,7 +124,7 @@ public class FtpSession implements Session { + "'. Server replied with: " + this.client.getReplyString()); } if (logger.isInfoEnabled()) { - logger.info("File has been successfully transfered to: " + path); + logger.info("File has been successfully transferred to: " + path); } } @@ -196,7 +197,8 @@ public class FtpSession implements Session { Assert.hasText(path, "'path' must not be empty"); String currentWorkingPath = this.client.printWorkingDirectory(); - Assert.state(currentWorkingPath != null, "working directory cannot be determined, therefore exists check can not be completed"); + Assert.state(currentWorkingPath != null, + "working directory cannot be determined, therefore exists check can not be completed"); boolean exists = false; try { diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml index 2d2925e481..efb5d84bfe 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml @@ -41,8 +41,9 @@ @@ -169,5 +170,22 @@ + + + + + diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 7a83af0d48..683258e2d7 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2015 the original author or authors. + * Copyright 2013-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,17 @@ package org.springframework.integration.ftp.outbound; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.isOneOf; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -44,6 +49,7 @@ import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.hamcrest.Matchers; import org.junit.Before; @@ -57,6 +63,7 @@ import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.endpoint.SourcePollingChannelAdapter; import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.InputStreamCallback; @@ -139,6 +146,12 @@ public class FtpServerOutboundTests { @Autowired private DirectChannel inboundCallback; + @Autowired + private DirectChannel inboundLs; + + @Autowired + private SourcePollingChannelAdapter ftpInbound; + @Before public void setup() { this.ftpServer.recursiveDelete(ftpServer.getTargetLocalDirectory()); @@ -183,7 +196,7 @@ public class FtpServerOutboundTests { @SuppressWarnings("unchecked") public void testInt2866LocalDirectoryExpressionMGET() { String dir = "ftpSource/"; - this.inboundMGet.send(new GenericMessage(dir + "*.txt")); + this.inboundMGet.send(new GenericMessage("*.txt")); Message result = this.output.receive(1000); assertNotNull(result); List localFiles = (List) result.getPayload(); @@ -205,11 +218,29 @@ public class FtpServerOutboundTests { } } + @Test + @SuppressWarnings("unchecked") + public void testMGETOnNullDir() throws IOException { + Session session = ftpSessionFactory.getSession(); + ((FTPClient) session.getClientInstance()).changeWorkingDirectory("ftpSource"); + session.close(); + + this.inboundMGet.send(new GenericMessage("")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + + for (File file : localFiles) { + assertThat(file.getName(), isOneOf("localTarget1.txt", "localTarget2.txt")); + assertThat(file.getName(), not(containsString("null"))); + } + } + @Test @SuppressWarnings("unchecked") public void testInt3172LocalDirectoryExpressionMGETRecursive() { String dir = "ftpSource/"; - this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + this.inboundMGetRecursive.send(new GenericMessage("*")); Message result = this.output.receive(1000); assertNotNull(result); List localFiles = (List) result.getPayload(); @@ -399,17 +430,19 @@ public class FtpServerOutboundTests { @Test public void testMgetPartial() throws Exception { Session session = spyOnSession(); - doAnswer(new Answer() { + doAnswer(new Answer() { @Override - public String[] answer(InvocationOnMock invocation) throws Throwable { - String[] files = (String[]) invocation.callRealMethod(); + public FTPFile[] answer(InvocationOnMock invocation) throws Throwable { + FTPFile[] files = (FTPFile[]) invocation.callRealMethod(); // add an extra file where the get will fail files = Arrays.copyOf(files, files.length + 1); - files[files.length - 1] = "bogus.txt"; + FTPFile bogusFile = new FTPFile(); + bogusFile.setName("bogus.txt"); + files[files.length - 1] = bogusFile; return files; } - }).when(session).listNames("ftpSource/subFtpSource/*"); + }).when(session).list("ftpSource/subFtpSource/*"); String dir = "ftpSource/subFtpSource/"; try { this.inboundMGet.send(new GenericMessage(dir + "*")); @@ -547,6 +580,51 @@ public class FtpServerOutboundTests { assertEquals("FOO", receive.getPayload()); } + @Test + @SuppressWarnings("unchecked") + public void testLsForNullDir() throws IOException { + Session session = ftpSessionFactory.getSession(); + ((FTPClient) session.getClientInstance()).changeWorkingDirectory("ftpSource"); + session.close(); + + this.inboundLs.send(new GenericMessage("foo")); + Message receive = this.output.receive(10000); + assertNotNull(receive); + assertThat(receive.getPayload(), instanceOf(List.class)); + List files = (List) receive.getPayload(); + assertEquals(2, files.size()); + assertThat(files, containsInAnyOrder("ftpSource1.txt", "ftpSource2.txt")); + + FTPFile[] ftpFiles = ftpSessionFactory.getSession().list(null); + for (FTPFile ftpFile : ftpFiles) { + if (!ftpFile.isDirectory()) { + assertTrue(files.contains(ftpFile.getName())); + } + } + } + + @Test + public void testInboundChannelAdapterWithNullDir() throws IOException { + Session session = ftpSessionFactory.getSession(); + ((FTPClient) session.getClientInstance()).changeWorkingDirectory("ftpSource"); + session.close(); + this.ftpInbound.start(); + + Message message = this.output.receive(10000); + assertNotNull(message); + assertThat(message.getPayload(), instanceOf(File.class)); + assertEquals("ftpSource1.txt", ((File) message.getPayload()).getName()); + + message = this.output.receive(10000); + assertNotNull(message); + assertThat(message.getPayload(), instanceOf(File.class)); + assertEquals("ftpSource2.txt", ((File) message.getPayload()).getName()); + + assertNull(this.output.receive(10)); + + this.ftpInbound.stop(); + } + public static class SortingFileListFilter implements FileListFilter { @Override diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index b3abbb54c3..949996f95c 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -191,6 +191,9 @@ Starting with _version 4.2_, you can specify `remote-directory-expression` inste you to dynamically determine the directory on each poll. e.g `remote-directory-expression="@myBean.determineRemoteDir()"`. +Starting with _version 4.3_, the `remote-directory`/`remote-directory-expression` attributes can be omitted assuming `null`. +In this case, according to the FTP protocol, the Client working directory is used as a default remote directory. + Sometimes file filtering based on the simple pattern specified via `filename-pattern` attribute might not be sufficient. If this is the case, you can use the `filename-regex` attribute to specify a Regular Expression (e.g. `filename-regex=".*\.test$"`). @@ -255,7 +258,10 @@ Here is an example that uses a custom Filter implementation. _Poller configuration notes for the inbound FTP adapter_ -The job of the inbound FTP adapter consists of two tasks: _1) Communicate with a remote server in order to transfer files from a remote directory to a local directory.__2) For each transferred file, generate a Message with that file as a payload and send it to the channel identified by the 'channel' attribute._ That is why they are called 'channel-adapters' rather than just 'adapters'. +The job of the inbound FTP adapter consists of two tasks: +_1) Communicate with a remote server in order to transfer files from a remote directory to a local directory._ +_2) For each transferred file, generate a Message with that file as a payload and send it to the channel identified by the 'channel' attribute._ +That is why they are called 'channel-adapters' rather than just 'adapters'. The main job of such an adapter is to generate a Message to be sent to a Message Channel. Essentially, the second task mentioned above takes precedence in such a way that *IF* your local directory already has one or more files it will first generate Messages from those, and *ONLY* when all local files have been processed, will it initiate the remote communication to retrieve more files. @@ -407,6 +413,13 @@ If the `-dirs` option is included, each recursive directory is also returned as In this case, it is recommended that the `-1` is not used because you would not be able to determine files Vs. directories, which is achievable using the `FileInfo` objects. +Starting with _version 4.3_, the `FtpSession` supports `null` for the `list()` and `listNames()` methods, +therefore the `expression` attribute can be omitted. +From Java perspective there are two new constructor without `expression` argument for convenience. +The `null` for `LS` command is treated as an Client working directory according to the FTP protocol. +The working directory can be set via the `FTPClient.changeWorkingDirectory()` function when you extend the +`DefaultFtpSessionFactory` and implement `postProcessClientBeforeConnect()` callback. + *get* _get_ retrieves a remote file and supports the following option: diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 934f139a08..98e0d9cec1 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -114,6 +114,14 @@ See <> for more information. A new factory bean is provided to simplify the configuration of Jsch proxies for SFTP. See <> for more information. +==== FTP Changes + +The `FtpSession` now supports `null` for the `list()` and `listNames()` method, since it is possible by the +underlying FTP Client. +With that the `FtpOutboundGateway` can now be configured without `remoteDirectory` expression. +And the `` can be configured without `remote-directory`/`remote-directory-expression`. +See <> for more information. + ==== Router Changes The `ErrorMessageExceptionTypeRouter` supports now the `Exception` superclass mappings to avoid duplication