Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-3554: Eval remote dir on each synchToLocal (#3556)

Browse Source 
* GH-3554: Eval remote dir on each synchToLocal

Fixes https://github.com/spring-projects/spring-integration/issues/3554

The `remoteDirectoryExpression` was introduced into an
`AbstractInboundFileSynchronizer` to let end-user to evaluate a remote directory
on each poll (essentially on each `synchronizeToLocalDirectory()` call).
The fix for the https://jira.spring.io/browse/INT-4491 introduced a regression
when such an expression was evaluated only once when we call a `setRemoteDirectory()`.
So, an original purpose of this option was lost and we don't get an actual
remote dir on each poll

* Remove `evaluateRemoteDirectory()` method and its usage since it doesn't
reflect expectation of the remote dir expression property
* Reinstate the `remoteDirectoryExpression` evaluation in the
`synchronizeToLocalDirectory()`
* Propagate the result of that expression into further methods for copying
files
* Remove setting of the `remoteDirectory` variable into a global `EvaluationContext`
of the `AbstractInboundFileSynchronizer` instance since it is not thread-safe
* Instead create an `EvaluationContext` locally for each `synchronizeToLocalDirectory()`
call and set the `remoteDirectory` variable into this scoped instances
* Generate a local file name from the `localFilenameGeneratorExpression`
against locally created `EvaluationContext` with the mentioned `remoteDirectory` variable
* Cover the expected functionality with a unit-test

**Cherry-pick to `5.4.x` & `5.3.x`**

* * Fix `testRemoteDirectoryRefreshedOnEachSynchronization` according PR review
This commit is contained in:
Artem Bilan
2021-04-27 16:36:54 -04:00
committed by GitHub
parent 0dfbfa0677
commit 0d7bbeaa4e
3 changed files with 102 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2020 the original author or authors.
* Copyright 2002-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -102,11 +102,6 @@ public abstract class AbstractInboundFileSynchronizer<F>
*/
private Expression remoteDirectoryExpression;
/**
* The current evaluation of the expression.
*/
private String evaluatedRemoteDirectory;
/**
* An {@link FileListFilter} that runs against the <em>remote</em> file system view.
*/
@@ -203,7 +198,6 @@ public abstract class AbstractInboundFileSynchronizer<F>
*/
public void setRemoteDirectory(String remoteDirectory) {
this.remoteDirectoryExpression = new LiteralExpression(remoteDirectory);
evaluateRemoteDirectory();
}
/**
@@ -229,7 +223,6 @@ public abstract class AbstractInboundFileSynchronizer<F>
protected final void doSetRemoteDirectoryExpression(Expression expression) {
Assert.notNull(expression, "'remoteDirectoryExpression' must not be null");
this.remoteDirectoryExpression = expression;
evaluateRemoteDirectory();
}
/**
@@ -298,7 +291,6 @@ public abstract class AbstractInboundFileSynchronizer<F>
if (this.evaluationContext == null) {
this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.beanFactory);
}
evaluateRemoteDirectory();
if (!StringUtils.hasText(this.metadataStorePrefix)) {
this.metadataStorePrefix = this.name;
}
@@ -341,26 +333,27 @@ public abstract class AbstractInboundFileSynchronizer<F>
}
return;
}
String remoteDirectory = this.remoteDirectoryExpression.getValue(this.evaluationContext, String.class);
if (this.logger.isTraceEnabled()) {
this.logger.trace("Synchronizing " + this.evaluatedRemoteDirectory + " to " + localDirectory);
this.logger.trace("Synchronizing " + remoteDirectory + " to " + localDirectory);
}
try {
int transferred = this.remoteFileTemplate.execute(session ->
transferFilesFromRemoteToLocal(localDirectory, maxFetchSize, session));
transferFilesFromRemoteToLocal(remoteDirectory, localDirectory, maxFetchSize, session));
if (this.logger.isDebugEnabled()) {
this.logger.debug(transferred + " files transferred from '" + this.evaluatedRemoteDirectory + "'");
this.logger.debug(transferred + " files transferred from '" + remoteDirectory + "'");
}
}
catch (Exception e) {
throw new MessagingException("Problem occurred while synchronizing '"
+ this.evaluatedRemoteDirectory + "' to local directory", e);
+ remoteDirectory + "' to local directory", e);
}
}
private Integer transferFilesFromRemoteToLocal(File localDirectory, int maxFetchSize, Session<F> session)
throws IOException {
private Integer transferFilesFromRemoteToLocal(String remoteDirectory, File localDirectory,
int maxFetchSize, Session<F> session) throws IOException {
F[] files = session.list(this.evaluatedRemoteDirectory);
F[] files = session.list(remoteDirectory);
if (!ObjectUtils.isEmpty(files)) {
files = FileUtils.purgeUnwantedElements(files, e -> !isFile(e), this.comparator);
}
@@ -372,6 +365,12 @@ public abstract class AbstractInboundFileSynchronizer<F>
int copied = filteredFiles.size();
int accepted = 0;
EvaluationContext localFileEvaluationContext = null;
if (this.localFilenameGeneratorExpression != null) {
localFileEvaluationContext = ExpressionUtils.createStandardEvaluationContext(this.beanFactory);
localFileEvaluationContext.setVariable("remoteDirectory", remoteDirectory);
}
for (F file : filteredFiles) {
if (filteringOneByOne) {
if ((maxFetchSize < 0 || accepted < maxFetchSize) && this.filter
@@ -383,7 +382,9 @@ public abstract class AbstractInboundFileSynchronizer<F>
copied--;
}
}
copied = copyIfNotNull(localDirectory, session, filteringOneByOne, filteredFiles, copied, file);
copied =
copyIfNotNull(remoteDirectory, localDirectory, localFileEvaluationContext, session,
filteringOneByOne, filteredFiles, copied, file);
}
return copied;
}
@@ -392,13 +393,16 @@ public abstract class AbstractInboundFileSynchronizer<F>
}
}
private int copyIfNotNull(File localDirectory, Session<F> session, boolean filteringOneByOne, List<F> filteredFiles,
int copied, @Nullable F file) throws IOException {
private int copyIfNotNull(String remoteDirectory, File localDirectory,
@Nullable EvaluationContext localFileEvaluationContext, Session<F> session, boolean filteringOneByOne,
List<F> filteredFiles, int copied, @Nullable F file) throws IOException {
boolean renamedFailed = false;
try {
if (file != null && !copyFileToLocalDirectory(this.evaluatedRemoteDirectory, file,
localDirectory, session)) {
if (file != null &&
!copyFileToLocalDirectory(remoteDirectory, localFileEvaluationContext, file, localDirectory,
session)) {
renamedFailed = true;
}
}
@@ -440,11 +444,12 @@ public abstract class AbstractInboundFileSynchronizer<F>
}
}
protected boolean copyFileToLocalDirectory(String remoteDirectoryPath, F remoteFile, // NOSONAR
File localDirectory, Session<F> session) throws IOException {
protected boolean copyFileToLocalDirectory(String remoteDirectoryPath, // NOSONAR
@Nullable EvaluationContext localFileEvaluationContext, F remoteFile, File localDirectory,
Session<F> session) throws IOException {
String remoteFileName = getFilename(remoteFile);
String localFileName = generateLocalFileName(remoteFileName);
String localFileName = generateLocalFileName(remoteFileName, localFileEvaluationContext);
String remoteFilePath = remoteDirectoryPath != null
? (remoteDirectoryPath + this.remoteFileSeparator + remoteFileName)
: remoteFileName;
@@ -567,22 +572,16 @@ public abstract class AbstractInboundFileSynchronizer<F>
return renamed;
}
private String generateLocalFileName(String remoteFileName) {
private String generateLocalFileName(String remoteFileName,
@Nullable EvaluationContext localFileEvaluationContext) {
if (this.localFilenameGeneratorExpression != null) {
return this.localFilenameGeneratorExpression.getValue(this.evaluationContext, remoteFileName,
return this.localFilenameGeneratorExpression.getValue(localFileEvaluationContext, remoteFileName,
String.class);
}
return remoteFileName;
}
protected void evaluateRemoteDirectory() {
if (this.evaluationContext != null) {
this.evaluatedRemoteDirectory = this.remoteDirectoryExpression.getValue(this.evaluationContext,
String.class);
this.evaluationContext.setVariable("remoteDirectory", this.evaluatedRemoteDirectory);
}
}
/**
* Obtain a metadata for remote file associated with the provided local file.
* @param localFile the local file to retrieve metadata for.