GH-75: Fix remote directory representation

Resolves spring-projects/spring-integration-aws#75

Since we can build S3 entity key any deep path,
e.g. `my_bucket/foo/bar/baz/file.name` and S3 Object `key` representation
is exactly the whole path without bucket name, we should treat only bucket
as a remote dir; the key should be as a file name

* Change `S3InboundFileSynchronizer` logic to extract bucket name before
performing `copyFileToLocalDirectory()`
* Change `S3StreamingMessageSource` to override the `S3FileInfo.remoteDirectory`
to the only bucket name after `poll()`.
* Use for both cases a new `S3Session.normalizeBucketName()` method
* Upgrade to Gradle 4.0 and some other `build.gradle` polishing
This commit is contained in:
Artem Bilan
2017-07-10 20:32:39 -04:00
parent 54782d13c5
commit d187c838b6
8 changed files with 45 additions and 21 deletions

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2016 the original author or authors.
* Copyright 2016-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,8 +16,12 @@
package org.springframework.integration.aws.inbound;
import java.io.File;
import java.io.IOException;
import org.springframework.expression.Expression;
import org.springframework.expression.common.LiteralExpression;
import org.springframework.integration.aws.support.S3Session;
import org.springframework.integration.aws.support.S3SessionFactory;
import org.springframework.integration.aws.support.filters.S3PersistentAcceptOnceFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
@@ -79,4 +83,11 @@ public class S3InboundFileSynchronizer extends AbstractInboundFileSynchronizer<S
return file.getLastModified().getTime();
}
@Override
protected void copyFileToLocalDirectory(String remoteDirectoryPath, S3ObjectSummary remoteFile,
File localDirectory, Session<S3ObjectSummary> session) throws IOException {
super.copyFileToLocalDirectory(((S3Session) session).normalizeBucketName(remoteDirectoryPath),
remoteFile, localDirectory, session);
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2016 the original author or authors.
* Copyright 2016-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import java.util.Comparator;
import java.util.List;
import org.springframework.integration.aws.support.S3FileInfo;
import org.springframework.integration.aws.support.S3Session;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
import org.springframework.integration.file.remote.RemoteFileTemplate;
@@ -32,6 +33,8 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
* A {@link AbstractRemoteFileStreamingMessageSource} implementation for the Amazon S3.
*
* @author Christian Tzolov
* @author Artem Bilan
*
* @since 1.1
*/
public class S3StreamingMessageSource extends AbstractRemoteFileStreamingMessageSource<S3ObjectSummary> {
@@ -41,13 +44,13 @@ public class S3StreamingMessageSource extends AbstractRemoteFileStreamingMessage
}
public S3StreamingMessageSource(RemoteFileTemplate<S3ObjectSummary> template,
Comparator<AbstractFileInfo<S3ObjectSummary>> comparator) {
Comparator<AbstractFileInfo<S3ObjectSummary>> comparator) {
super(template, comparator);
}
@Override
protected List<AbstractFileInfo<S3ObjectSummary>> asFileInfoList(Collection<S3ObjectSummary> collection) {
List<AbstractFileInfo<S3ObjectSummary>> canonicalFiles = new ArrayList<AbstractFileInfo<S3ObjectSummary>>();
for (S3ObjectSummary s3ObjectSummary : collection) {
canonicalFiles.add(new S3FileInfo(s3ObjectSummary));
@@ -60,4 +63,14 @@ public class S3StreamingMessageSource extends AbstractRemoteFileStreamingMessage
return "aws:s3-inbound-streaming-channel-adapter";
}
@Override
protected AbstractFileInfo<S3ObjectSummary> poll() {
AbstractFileInfo<S3ObjectSummary> file = super.poll();
if (file != null) {
S3Session s3Session = (S3Session) getRemoteFileTemplate().getSession();
file.setRemoteDirectory(s3Session.normalizeBucketName(file.getRemoteDirectory()));
}
return file;
}
}

View File

@@ -228,6 +228,10 @@ public class S3Session implements Session<S3ObjectSummary> {
return this.amazonS3;
}
public String normalizeBucketName(String path) {
return splitPathToBucketAndKey(path, false)[0];
}
private String[] splitPathToBucketAndKey(String path, boolean requireKey) {
Assert.hasText(path, "'path' must not be empty String.");

View File

@@ -162,7 +162,7 @@ public class S3StreamingChannelAdapterTests {
S3RemoteFileTemplate s3FileTemplate = new S3RemoteFileTemplate(s3SessionFactory);
S3StreamingMessageSource s3MessageSource = new S3StreamingMessageSource(s3FileTemplate,
Comparator.comparing(FileInfo::getFilename));
s3MessageSource.setRemoteDirectory("/" + S3_BUCKET);
s3MessageSource.setRemoteDirectory("/" + S3_BUCKET + "/subdir");
s3MessageSource.setFilter(new S3PersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "streaming"));
return s3MessageSource;