GH-75: Fix remote directory representation
Resolves spring-projects/spring-integration-aws#75 Since we can build S3 entity key any deep path, e.g. `my_bucket/foo/bar/baz/file.name` and S3 Object `key` representation is exactly the whole path without bucket name, we should treat only bucket as a remote dir; the key should be as a file name * Change `S3InboundFileSynchronizer` logic to extract bucket name before performing `copyFileToLocalDirectory()` * Change `S3StreamingMessageSource` to override the `S3FileInfo.remoteDirectory` to the only bucket name after `poll()`. * Use for both cases a new `S3Session.normalizeBucketName()` method * Upgrade to Gradle 4.0 and some other `build.gradle` polishing
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
* Copyright 2016-2017 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,8 +16,12 @@
|
||||
|
||||
package org.springframework.integration.aws.inbound;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.springframework.expression.Expression;
|
||||
import org.springframework.expression.common.LiteralExpression;
|
||||
import org.springframework.integration.aws.support.S3Session;
|
||||
import org.springframework.integration.aws.support.S3SessionFactory;
|
||||
import org.springframework.integration.aws.support.filters.S3PersistentAcceptOnceFileListFilter;
|
||||
import org.springframework.integration.file.filters.FileListFilter;
|
||||
@@ -79,4 +83,11 @@ public class S3InboundFileSynchronizer extends AbstractInboundFileSynchronizer<S
|
||||
return file.getLastModified().getTime();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void copyFileToLocalDirectory(String remoteDirectoryPath, S3ObjectSummary remoteFile,
|
||||
File localDirectory, Session<S3ObjectSummary> session) throws IOException {
|
||||
super.copyFileToLocalDirectory(((S3Session) session).normalizeBucketName(remoteDirectoryPath),
|
||||
remoteFile, localDirectory, session);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
* Copyright 2016-2017 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -22,6 +22,7 @@ import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.integration.aws.support.S3FileInfo;
|
||||
import org.springframework.integration.aws.support.S3Session;
|
||||
import org.springframework.integration.file.remote.AbstractFileInfo;
|
||||
import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
|
||||
import org.springframework.integration.file.remote.RemoteFileTemplate;
|
||||
@@ -32,6 +33,8 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
* A {@link AbstractRemoteFileStreamingMessageSource} implementation for the Amazon S3.
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
* @author Artem Bilan
|
||||
*
|
||||
* @since 1.1
|
||||
*/
|
||||
public class S3StreamingMessageSource extends AbstractRemoteFileStreamingMessageSource<S3ObjectSummary> {
|
||||
@@ -41,13 +44,13 @@ public class S3StreamingMessageSource extends AbstractRemoteFileStreamingMessage
|
||||
}
|
||||
|
||||
public S3StreamingMessageSource(RemoteFileTemplate<S3ObjectSummary> template,
|
||||
Comparator<AbstractFileInfo<S3ObjectSummary>> comparator) {
|
||||
Comparator<AbstractFileInfo<S3ObjectSummary>> comparator) {
|
||||
|
||||
super(template, comparator);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<AbstractFileInfo<S3ObjectSummary>> asFileInfoList(Collection<S3ObjectSummary> collection) {
|
||||
|
||||
List<AbstractFileInfo<S3ObjectSummary>> canonicalFiles = new ArrayList<AbstractFileInfo<S3ObjectSummary>>();
|
||||
for (S3ObjectSummary s3ObjectSummary : collection) {
|
||||
canonicalFiles.add(new S3FileInfo(s3ObjectSummary));
|
||||
@@ -60,4 +63,14 @@ public class S3StreamingMessageSource extends AbstractRemoteFileStreamingMessage
|
||||
return "aws:s3-inbound-streaming-channel-adapter";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AbstractFileInfo<S3ObjectSummary> poll() {
|
||||
AbstractFileInfo<S3ObjectSummary> file = super.poll();
|
||||
if (file != null) {
|
||||
S3Session s3Session = (S3Session) getRemoteFileTemplate().getSession();
|
||||
file.setRemoteDirectory(s3Session.normalizeBucketName(file.getRemoteDirectory()));
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -228,6 +228,10 @@ public class S3Session implements Session<S3ObjectSummary> {
|
||||
return this.amazonS3;
|
||||
}
|
||||
|
||||
public String normalizeBucketName(String path) {
|
||||
return splitPathToBucketAndKey(path, false)[0];
|
||||
}
|
||||
|
||||
private String[] splitPathToBucketAndKey(String path, boolean requireKey) {
|
||||
Assert.hasText(path, "'path' must not be empty String.");
|
||||
|
||||
|
||||
@@ -162,7 +162,7 @@ public class S3StreamingChannelAdapterTests {
|
||||
S3RemoteFileTemplate s3FileTemplate = new S3RemoteFileTemplate(s3SessionFactory);
|
||||
S3StreamingMessageSource s3MessageSource = new S3StreamingMessageSource(s3FileTemplate,
|
||||
Comparator.comparing(FileInfo::getFilename));
|
||||
s3MessageSource.setRemoteDirectory("/" + S3_BUCKET);
|
||||
s3MessageSource.setRemoteDirectory("/" + S3_BUCKET + "/subdir");
|
||||
s3MessageSource.setFilter(new S3PersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "streaming"));
|
||||
|
||||
return s3MessageSource;
|
||||
|
||||
Reference in New Issue
Block a user