Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-8691: Add (S)FTP, SMB aged file filters

Browse Source 
Fixes https://github.com/spring-projects/spring-integration/issues/8691

* Remove setAge with TimeUnit
Turned out we don't need anymore since we're using Duration for age in FtpLastModifiedFileListFilter and SftpLastModifiedFileListFilter.
* Add changes to the docs
* Introduce AbstractLastModifiedFileListFilter
* Some code readability improvements
* Make language in the docs more official
This commit is contained in:
Adama Sorho
2023-09-02 00:56:31 -04:00
committed by Artem Bilan
parent f3d0441a38
commit 73ed3eeebd
14 changed files with 540 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractLastModifiedFileListFilter.java Normal file
View File

@@ -0,0 +1,128 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.filters;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import org.springframework.lang.Nullable;
/**
* The {@link FileListFilter} implementation to filter those files which
* lastModified is less than the {@link #age} in comparison
* with the {@link Instant#now()}.
* When {@link #discardCallback} is provided, it called for all the rejected files.
*
* @param <F> the file
*
* @author Adama Sorho
* @author Artem Bilan
*
* @since 6.2
*/
public abstract class AbstractLastModifiedFileListFilter<F> implements DiscardAwareFileListFilter<F> {
protected static final long ONE_SECOND = 1000;
private static final long DEFAULT_AGE = 60;
private Duration age = Duration.ofSeconds(DEFAULT_AGE);
@Nullable
private Consumer<F> discardCallback;
public AbstractLastModifiedFileListFilter() {
}
public AbstractLastModifiedFileListFilter(Duration age) {
this.age = age;
}
/**
* Set the age that files have to be before being passed by this filter.
* If lastModified plus {@link #age} is before the {@link Instant#now()}, the file
* is filtered.
* Defaults to 60 seconds.
* @param age the Duration.
*/
public void setAge(Duration age) {
this.age = age;
}
/**
* Set the age that files have to be before being passed by this filter.
* If lastModified plus {@link #age} is before the {@link Instant#now()}, the file
* is filtered.
* Defaults to 60 seconds.
* @param age the age in seconds.
*/
public void setAge(long age) {
setAge(Duration.ofSeconds(age));
}
@Override
public void addDiscardCallback(@Nullable Consumer<F> discardCallback) {
this.discardCallback = discardCallback;
}
@Override
public List<F> filterFiles(F[] files) {
List<F> list = new ArrayList<>();
Instant now = Instant.now();
for (F file: files) {
if (fileIsAged(file, now)) {
list.add(file);
}
else if (this.discardCallback != null) {
this.discardCallback.accept(file);
}
}
return list;
}
@Override
public boolean accept(F file) {
if (fileIsAged(file, Instant.now())) {
return true;
}
else if (this.discardCallback != null) {
this.discardCallback.accept(file);
}
return false;
}
private boolean fileIsAged(F file, Instant now) {
return getLastModified(file).plus(this.age).isBefore(now);
}
@Override
public boolean supportsSingleFileFiltering() {
return true;
}
protected Duration getAgeDuration() {
return this.age;
}
protected abstract Instant getLastModified(F remoteFile);
}

97
spring-integration-file/src/main/java/org/springframework/integration/file/filters/LastModifiedFileListFilter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015-2022 the original author or authors.
* Copyright 2015-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,51 +18,41 @@ package org.springframework.integration.file.filters;
import java.io.File;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import org.springframework.lang.Nullable;
/**
* The {@link FileListFilter} implementation to filter those files which
* {@link File#lastModified()} is less than the {@link #age} in comparison
* {@link File#lastModified()} is less than the age in comparison
* with the current time.
* <p>
* The resolution is done in seconds.
* <p>
* When {@link #discardCallback} is provided, it called for all the
* When discardCallback {@link #addDiscardCallback(Consumer)} is provided, it called for all the
* rejected files.
*
* @author Gary Russell
* @author Artem Bilan
* @author Adama Sorho
*
* @since 4.2
*
*/
public class LastModifiedFileListFilter implements DiscardAwareFileListFilter<File> {
private static final long ONE_SECOND = 1000;
private static final long DEFAULT_AGE = 60;
private volatile long age = DEFAULT_AGE;
@Nullable
private Consumer<File> discardCallback;
public class LastModifiedFileListFilter extends AbstractLastModifiedFileListFilter<File> {
public LastModifiedFileListFilter() {
super();
}
/**
* Construct a {@link LastModifiedFileListFilter} instance with provided {@link #age}.
* Construct a {@link LastModifiedFileListFilter} instance with provided age.
* Defaults to 60 seconds.
* @param age the age in seconds.
* @since 5.0
*/
public LastModifiedFileListFilter(long age) {
this.age = age;
super(Duration.ofSeconds(age));
}
/**
@@ -72,76 +62,25 @@ public class LastModifiedFileListFilter implements DiscardAwareFileListFilter<Fi
* Defaults to 60 seconds.
* @param age the age
* @param unit the timeUnit.
* @deprecated since 6.2 in favor of {@link #setAge(Duration)}
*/
@Deprecated(since = "6.2", forRemoval = true)
public void setAge(long age, TimeUnit unit) {
this.age = unit.toSeconds(age);
setAge(unit.toSeconds(age));
}
/**
* Set the age that files have to be before being passed by this filter.
* If {@link File#lastModified()} plus age is greater than the current time, the file
* is filtered. The resolution is seconds.
* Defaults to 60 seconds.
* @param age the age
* @since 5.1.3
* @return the age in seconds.
* @deprecated since 6.2 in favor of {@link #getAgeDuration()}
*/
public void setAge(Duration age) {
setAge(age.getSeconds());
}
/**
* Set the age that files have to be before being passed by this filter.
* If {@link File#lastModified()} plus age is greater than the current time, the file
* is filtered. The resolution is seconds.
* Defaults to 60 seconds.
* @param age the age
*/
public void setAge(long age) {
setAge(age, TimeUnit.SECONDS);
}
@Deprecated(since = "6.2", forRemoval = true)
public long getAge() {
return this.age;
return getAgeDuration().getSeconds();
}
@Override
public void addDiscardCallback(@Nullable Consumer<File> discardCallbackToSet) {
this.discardCallback = discardCallbackToSet;
}
@Override
public List<File> filterFiles(File[] files) {
List<File> list = new ArrayList<>();
long now = System.currentTimeMillis() / ONE_SECOND;
for (File file : files) {
if (fileIsAged(file, now)) {
list.add(file);
}
else if (this.discardCallback != null) {
this.discardCallback.accept(file);
}
}
return list;
}
@Override
public boolean accept(File file) {
if (fileIsAged(file, System.currentTimeMillis() / ONE_SECOND)) {
return true;
}
else if (this.discardCallback != null) {
this.discardCallback.accept(file);
}
return false;
}
private boolean fileIsAged(File file, long now) {
return file.lastModified() / ONE_SECOND + this.age <= now;
}
@Override
public boolean supportsSingleFileFiltering() {
return true;
protected Instant getLastModified(File file) {
return Instant.ofEpochSecond(file.lastModified() / ONE_SECOND);
}
}

23
spring-integration-file/src/test/java/org/springframework/integration/file/filters/LastModifiedFileListFilterTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015-2022 the original author or authors.
* Copyright 2015-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,37 +18,38 @@ package org.springframework.integration.file.filters;
import java.io.File;
import java.io.FileOutputStream;
import java.util.concurrent.TimeUnit;
import java.time.Duration;
import java.time.Instant;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Gary Russell
* @author Artem Bilan
* @author Adama Sorho
*
* @since 4.2
*
*/
public class LastModifiedFileListFilterTests {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@TempDir
public File folder;
@Test
public void testAge() throws Exception {
LastModifiedFileListFilter filter = new LastModifiedFileListFilter();
filter.setAge(60, TimeUnit.SECONDS);
File foo = this.folder.newFile();
filter.setAge(60);
File foo = new File(folder, "test.tmp");
FileOutputStream fileOutputStream = new FileOutputStream(foo);
fileOutputStream.write("x".getBytes());
fileOutputStream.close();
assertThat(filter.filterFiles(new File[] {foo})).hasSize(0);
assertThat(filter.accept(foo)).isFalse();
// Make a file as of yesterday's
foo.setLastModified(System.currentTimeMillis() - 1000 * 60 * 60 * 24);
foo.setLastModified(Instant.now().minus(Duration.ofDays(1)).toEpochMilli());
assertThat(filter.filterFiles(new File[] {foo})).hasSize(1);
assertThat(filter.accept(foo)).isTrue();
}