GH-3488: Fix Persistent Filters with Recursion

Resolves https://github.com/spring-projects/spring-integration/issues/3488

Resolves two problems:

- When changes are made deep in the directory tree, they were not detected because
  the directory is in the metadata store and only passes the filter if a file
  immediately under it is changed, changing the directory's timestamp.

This is solved by subclassing `AbstractDirectoryAwareFileListFilter`, allowing its
`alwaysAcceptDirectories` property to be set.

- Only the filename was used as a metadata key; causing problems if a file with the
  same name appears multiple times in the tree.

This is solved with a new property on `AbstractDirectoryAwareFileListFilter` used by
the gateways to determine whether to filter the raw file names returned by the session
(previous behavior) or the full path relative to the root directory.

**cherry-pick to 5.4.x, 5.3.x**

* Some code style clean up

# Conflicts:
#	spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
#	src/reference/asciidoc/whats-new.adoc

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpPersistentAcceptOnceFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2019 the original author or authors.
+ * Copyright 2013-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import com.jcraft.jsch.ChannelSftp.LsEntry;
  *
  * @author Gary Russell
  * @author David Liu
+ *
  * @since 3.0
  *
  */
@@ -46,4 +47,9 @@ public class SftpPersistentAcceptOnceFileListFilter extends AbstractPersistentAc
 		return file.getFilename();
 	}
 
+	@Override
+	protected boolean isDirectory(LsEntry file) {
+		return file.getAttrs().isDir();
+	}
+
 }

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml

@@ -87,7 +87,7 @@
 							  expression="payload"
 							  command-options="-R"
 							  mode="REPLACE_IF_MODIFIED"
-							  filter="dotStarDotTxtFilter"
+							  filter="persistentFilter"
 							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
 							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
@@ -98,6 +98,19 @@
 		<property name="alwaysAcceptDirectories" value="true" />
 	</bean>
 
+	<bean id="persistentFilter" class="org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter">
+		<constructor-arg ref="store"/>
+		<constructor-arg value="test"/>
+		<property name="forRecursion" value="true"/>
+		<property name="flushOnUpdate" value="true"/>
+	</bean>
+
+	<bean id="store" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore">
+		<property name="baseDirectory"
+			value="#{T(org.springframework.integration.file.remote.RemoteFileTestSupport).getScratchTempFolder().absolutePath}"/>
+	</bean>
+
+
 	<int:channel id="inboundMGetRecursiveFiltered"/>
 
 	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java

@@ -25,7 +25,9 @@ import static org.mockito.Mockito.verify;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.io.UncheckedIOException;
@@ -52,6 +54,7 @@ import org.springframework.integration.file.remote.MessageSessionCallback;
 import org.springframework.integration.file.remote.session.CachingSessionFactory;
 import org.springframework.integration.file.remote.session.Session;
 import org.springframework.integration.file.remote.session.SessionFactory;
+import org.springframework.integration.metadata.PropertiesPersistingMetadataStore;
 import org.springframework.integration.sftp.SftpTestSupport;
 import org.springframework.integration.sftp.server.ApacheMinaSftpEvent;
 import org.springframework.integration.sftp.server.DirectoryCreatedEvent;
@@ -147,6 +150,9 @@ public class SftpServerOutboundTests extends SftpTestSupport {
 	@Autowired
 	private SftpRemoteFileTemplate template;
 
+	@Autowired
+	private PropertiesPersistingMetadataStore store;
+
 	@BeforeEach
 	public void setup() {
 		this.config.targetLocalDirectoryName = getTargetLocalDirectoryName();
@@ -320,6 +326,17 @@ public class SftpServerOutboundTests extends SftpTestSupport {
 						" sftpSource1.txt",
 						"sftpSource2.txt",
 						"subSftpSource/subSftpSource1.txt");
+		File newDeepFile = new File(this.sourceRemoteDirectory + "/subSftpSource/subSftpSource2.txt");
+		OutputStream fos = new FileOutputStream(newDeepFile);
+		fos.write("test".getBytes());
+		fos.close();
+		this.inboundLSRecursiveNoDirs.send(new GenericMessage<Object>(dir));
+		result = this.output.receive(1000);
+		assertThat(result).isNotNull();
+		files = (List<SftpFileInfo>) result.getPayload();
+		assertThat(files).hasSize(1);
+		assertThat(files.get(0).getFilename()).isEqualTo("subSftpSource/subSftpSource2.txt");
+		assertThat(this.store.get("testsubSftpSource/subSftpSource2.txt")).isNotNull();
 	}
 
 	private long setModifiedOnSource1() {