Add S3MessageHandler
Implement the `S3MessageHandler` based on the `TransferManager`. Provide `upload`, `download` and `copy` operations. Support single file upload/download as well as directory structure. The `S3MessageHandler` may behave as `one-way`, as well as `request-reply` component. The former is blocked, the second is async. See test-cases and JavaDocs for more info.
This commit is contained in:
Artem Bilan
@@ -0,0 +1,555 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.aws.outbound;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.springframework.expression.EvaluationContext;
|
||||
import org.springframework.expression.Expression;
|
||||
import org.springframework.expression.common.LiteralExpression;
|
||||
import org.springframework.integration.expression.ExpressionUtils;
|
||||
import org.springframework.integration.expression.ValueExpression;
|
||||
import org.springframework.integration.handler.AbstractReplyProducingMessageHandler;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageHandlingException;
|
||||
import org.springframework.messaging.MessageHeaders;
|
||||
import org.springframework.messaging.support.MessageBuilder;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.StreamUtils;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.event.ProgressEvent;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.internal.Mimetypes;
|
||||
import com.amazonaws.services.s3.model.AccessControlList;
|
||||
import com.amazonaws.services.s3.model.CannedAccessControlList;
|
||||
import com.amazonaws.services.s3.model.CopyObjectRequest;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.SetObjectAclRequest;
|
||||
import com.amazonaws.services.s3.transfer.ObjectMetadataProvider;
|
||||
import com.amazonaws.services.s3.transfer.PersistableTransfer;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListener;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListenerChain;
|
||||
import com.amazonaws.util.Md5Utils;
|
||||
|
||||
/**
|
||||
* The {@link AbstractReplyProducingMessageHandler} implementation for the Amazon S3 services.
|
||||
* <p>
|
||||
* The implementation is fully based on the {@link TransferManager} and support its {@code upload},
|
||||
* {@code download} and {@code copy} operations which can be determined by the provided
|
||||
* or evaluated via SpEL expression at runtime {@link S3MessageHandler.Command}.
|
||||
* <p>
|
||||
* This {@link AbstractReplyProducingMessageHandler} can behave as a "one-way" (by default) or
|
||||
* "request-reply" component according to the {@link #produceReply} constructor argument.
|
||||
* <p>
|
||||
* The "one-way" behavior is also blocking, which is achieved with the {@link Transfer#waitForException()}
|
||||
* invocation. Consider to use an async upstream hand off if this blocking behavior isn't appropriate.
|
||||
* <p>
|
||||
* The "request-reply" behavior is async and the {@link Transfer} result from the {@link TransferManager}
|
||||
* operation is sent to the {@link #outputChannel}, assuming the transfer progress observation in the
|
||||
* downstream flow.
|
||||
* <p>
|
||||
* The {@link S3ProgressListener} can be supplied to track the transfer progress.
|
||||
* Also the listener can be populated into the returned {@link Transfer} afterwards in the downstream flow.
|
||||
* * <p>
|
||||
* For the upload operation the {@link UploadMetadataProvider} callback can be supplied to populate required
|
||||
* {@link ObjectMetadata} options, as for a single entry, as well as for each file in directory to upload.
|
||||
* <p>
|
||||
* For the upload operation the {@link #objectAclExpression} can be provided to {@link AmazonS3#setObjectAcl}
|
||||
* after the successful transfer.
|
||||
* The supported SpEL result types are: {@link AccessControlList} or {@link CannedAccessControlList}.
|
||||
* <p>
|
||||
* For download operation the {@code payload} must be a {@link File} instance, representing a single file
|
||||
* for downloaded content or directory to download all files from the S3 virtual directory.
|
||||
* <p>
|
||||
* An S3 Object {@code key} for upload and download can be determined by the provided
|
||||
* {@link #keyExpression} or the {@link File#getName()} is used directly. The former has precedence.
|
||||
* <p>
|
||||
* For copy operation all {@link #keyExpression}, {@link #destinationBucketExpression} and
|
||||
* {@link #destinationKeyExpression} are required and must not evaluate to {@code null}.
|
||||
*
|
||||
* @author Artem Bilan
|
||||
*
|
||||
* @see TransferManager
|
||||
*/
|
||||
public class S3MessageHandler extends AbstractReplyProducingMessageHandler {
|
||||
|
||||
private final TransferManager transferManager;
|
||||
|
||||
private final boolean produceReply;
|
||||
|
||||
private final Expression bucketExpression;
|
||||
|
||||
private EvaluationContext evaluationContext;
|
||||
|
||||
private Expression keyExpression;
|
||||
|
||||
private Expression objectAclExpression;
|
||||
|
||||
private Expression destinationBucketExpression;
|
||||
|
||||
private Expression destinationKeyExpression;
|
||||
|
||||
private Expression commandExpression = new ValueExpression<>(Command.UPLOAD);
|
||||
|
||||
private S3ProgressListener s3ProgressListener;
|
||||
|
||||
private UploadMetadataProvider uploadMetadataProvider;
|
||||
|
||||
public S3MessageHandler(AmazonS3 amazonS3, String bucket) {
|
||||
this(amazonS3, bucket, false);
|
||||
}
|
||||
|
||||
public S3MessageHandler(AmazonS3 amazonS3, Expression bucketExpression) {
|
||||
this(amazonS3, bucketExpression, false);
|
||||
}
|
||||
|
||||
public S3MessageHandler(AmazonS3 amazonS3, String bucket, boolean produceReply) {
|
||||
this(new TransferManager(amazonS3), bucket, produceReply);
|
||||
Assert.notNull(amazonS3, "'amazonS3' must not be null");
|
||||
}
|
||||
|
||||
public S3MessageHandler(AmazonS3 amazonS3, Expression bucketExpression, boolean produceReply) {
|
||||
this(new TransferManager(amazonS3), bucketExpression, produceReply);
|
||||
Assert.notNull(amazonS3, "'amazonS3' must not be null");
|
||||
}
|
||||
|
||||
public S3MessageHandler(TransferManager transferManager, String bucket) {
|
||||
this(transferManager, bucket, false);
|
||||
}
|
||||
|
||||
public S3MessageHandler(TransferManager transferManager, Expression bucketExpression) {
|
||||
this(transferManager, bucketExpression, false);
|
||||
}
|
||||
|
||||
public S3MessageHandler(TransferManager transferManager, String bucket, boolean produceReply) {
|
||||
this(transferManager, new LiteralExpression(bucket), produceReply);
|
||||
Assert.notNull(bucket, "'bucket' must not be null");
|
||||
}
|
||||
|
||||
public S3MessageHandler(TransferManager transferManager, Expression bucketExpression, boolean produceReply) {
|
||||
Assert.notNull(transferManager, "'transferManager' must not be null");
|
||||
Assert.notNull(bucketExpression, "'bucketExpression' must not be null");
|
||||
this.transferManager = transferManager;
|
||||
this.bucketExpression = bucketExpression;
|
||||
this.produceReply = produceReply;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SpEL expression to evaluate S3 object key at runtime against {@code requestMessage}.
|
||||
* @param keyExpression the SpEL expression for S3 key.
|
||||
*/
|
||||
public void setKeyExpression(Expression keyExpression) {
|
||||
this.keyExpression = keyExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SpEL expression to evaluate S3 object ACL at runtime against {@code requestMessage}
|
||||
* for the {@code upload} operation.
|
||||
* @param objectAclExpression the SpEL expression for S3 object ACL.
|
||||
*/
|
||||
public void setObjectAclExpression(Expression objectAclExpression) {
|
||||
this.objectAclExpression = objectAclExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* Specify a {@link S3MessageHandler.Command} to perform against {@link TransferManager}.
|
||||
* @param command The {@link S3MessageHandler.Command} to use.
|
||||
* @see S3MessageHandler.Command
|
||||
*/
|
||||
public void setCommand(Command command) {
|
||||
Assert.notNull(command, "'command' must not be null");
|
||||
setCommandExpression(new ValueExpression<>(command));
|
||||
}
|
||||
|
||||
/**
|
||||
* The SpEL expression to evaluate the command to perform on {@link TransferManager}: {@code upload},
|
||||
* {@code download} or {@code copy}.
|
||||
* @param commandExpression the SpEL expression to evaluate the {@link TransferManager} operation.
|
||||
* @see Command
|
||||
*/
|
||||
public void setCommandExpression(Expression commandExpression) {
|
||||
Assert.notNull(commandExpression, "'commandExpression' must not be null");
|
||||
this.commandExpression = commandExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SpEL expression to evaluate the target S3 bucket for copy operation.
|
||||
* @param destinationBucketExpression the SpEL expression for destination bucket.
|
||||
* @see TransferManager#copy(String, String, String, String)
|
||||
*/
|
||||
public void setDestinationBucketExpression(Expression destinationBucketExpression) {
|
||||
this.destinationBucketExpression = destinationBucketExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SpEL expression to evaluate the target S3 key for copy operation.
|
||||
* @param destinationKeyExpression the SpEL expression for destination key.
|
||||
* @see TransferManager#copy(String, String, String, String)
|
||||
*/
|
||||
public void setDestinationKeyExpression(Expression destinationKeyExpression) {
|
||||
this.destinationKeyExpression = destinationKeyExpression;
|
||||
}
|
||||
|
||||
/**
|
||||
* Specify a {@link S3ProgressListener} for upload and download operations.
|
||||
* @param s3ProgressListener the {@link S3ProgressListener} to use.
|
||||
*/
|
||||
public void setS3ProgressListener(S3ProgressListener s3ProgressListener) {
|
||||
this.s3ProgressListener = s3ProgressListener;
|
||||
}
|
||||
|
||||
/**
|
||||
* Specify an {@link ObjectMetadata} callback to populate the metadata for upload operation,
|
||||
* e.g. {@code Content-MD5}, {@code Content-Type} or any other required options.
|
||||
* @param uploadMetadataProvider the {@link UploadMetadataProvider} to use for upload.
|
||||
*/
|
||||
public void setUploadMetadataProvider(UploadMetadataProvider uploadMetadataProvider) {
|
||||
this.uploadMetadataProvider = uploadMetadataProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doInit() {
|
||||
Assert.notNull(this.bucketExpression, "The 'bucketExpression' must not be null");
|
||||
super.doInit();
|
||||
this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object handleRequestMessage(Message<?> requestMessage) {
|
||||
Command command;
|
||||
if (this.commandExpression instanceof ValueExpression) {
|
||||
command = (Command) this.commandExpression.getValue();
|
||||
}
|
||||
else {
|
||||
command = this.commandExpression.getValue(this.evaluationContext, requestMessage, Command.class);
|
||||
Assert.state(command != null, "'commandExpression' ["
|
||||
+ this.commandExpression.getExpressionString()
|
||||
+ "] cannot evaluate to null.");
|
||||
}
|
||||
|
||||
Transfer transfer = null;
|
||||
|
||||
switch (command) {
|
||||
case UPLOAD:
|
||||
transfer = upload(requestMessage);
|
||||
break;
|
||||
|
||||
case DOWNLOAD:
|
||||
transfer = download(requestMessage);
|
||||
break;
|
||||
|
||||
case COPY:
|
||||
transfer = copy(requestMessage);
|
||||
break;
|
||||
}
|
||||
|
||||
if (this.produceReply) {
|
||||
return transfer;
|
||||
}
|
||||
else {
|
||||
try {
|
||||
AmazonClientException amazonClientException = transfer.waitForException();
|
||||
if (amazonClientException != null) {
|
||||
throw amazonClientException;
|
||||
}
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private Transfer upload(Message<?> requestMessage) {
|
||||
Object payload = requestMessage.getPayload();
|
||||
String bucketName = obtainBucket(requestMessage);
|
||||
|
||||
String key = null;
|
||||
if (this.keyExpression != null) {
|
||||
key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class);
|
||||
}
|
||||
|
||||
if (payload instanceof File && ((File) payload).isDirectory()) {
|
||||
File fileToUpload = (File) payload;
|
||||
if (key == null) {
|
||||
key = fileToUpload.getName();
|
||||
}
|
||||
return this.transferManager.uploadDirectory(bucketName, key, fileToUpload, true,
|
||||
new MessageHeadersObjectMetadataProvider(requestMessage.getHeaders()));
|
||||
}
|
||||
else {
|
||||
ObjectMetadata metadata = new ObjectMetadata();
|
||||
if (this.uploadMetadataProvider != null) {
|
||||
this.uploadMetadataProvider.populateMetadata(metadata, requestMessage);
|
||||
}
|
||||
InputStream inputStream;
|
||||
if (payload instanceof InputStream) {
|
||||
inputStream = (InputStream) payload;
|
||||
}
|
||||
else if (payload instanceof File) {
|
||||
File fileToUpload = (File) payload;
|
||||
if (key == null) {
|
||||
key = fileToUpload.getName();
|
||||
}
|
||||
try {
|
||||
inputStream = new FileInputStream(fileToUpload);
|
||||
|
||||
if (metadata.getContentLength() == 0) {
|
||||
metadata.setContentLength(fileToUpload.length());
|
||||
}
|
||||
if (metadata.getContentType() == null) {
|
||||
metadata.setContentType(Mimetypes.getInstance().getMimetype(fileToUpload));
|
||||
}
|
||||
|
||||
}
|
||||
catch (FileNotFoundException e) {
|
||||
throw new AmazonClientException(e);
|
||||
}
|
||||
}
|
||||
else if (payload instanceof byte[]) {
|
||||
inputStream = new ByteArrayInputStream((byte[]) payload);
|
||||
}
|
||||
else {
|
||||
throw new IllegalArgumentException("Unsupported payload type: ["
|
||||
+ payload.getClass()
|
||||
+ "]. The only supported payloads for the upload request are " +
|
||||
"java.io.File, java.io.InputStream, byte[] and PutObjectRequest.");
|
||||
}
|
||||
|
||||
Assert.state(key != null,
|
||||
"The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " +
|
||||
"Root object is: " + requestMessage);
|
||||
|
||||
if (key == null) {
|
||||
if (this.keyExpression != null) {
|
||||
throw new IllegalStateException("The 'keyExpression' ["
|
||||
+ this.keyExpression.getExpressionString()
|
||||
+ "] must not evaluate to null. Root object is: " + requestMessage);
|
||||
}
|
||||
else {
|
||||
throw new IllegalStateException("Specify a 'keyExpression' for non-java.io.File payloads");
|
||||
}
|
||||
}
|
||||
|
||||
if (metadata.getContentMD5() == null) {
|
||||
String contentMd5 = null;
|
||||
try {
|
||||
contentMd5 = Md5Utils.md5AsBase64(StreamUtils.copyToByteArray(inputStream));
|
||||
if (inputStream.markSupported()) {
|
||||
inputStream.reset();
|
||||
}
|
||||
metadata.setContentMD5(contentMd5);
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new MessageHandlingException(requestMessage, e);
|
||||
}
|
||||
}
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, metadata);
|
||||
|
||||
S3ProgressListener progressListener = this.s3ProgressListener;
|
||||
|
||||
if (this.objectAclExpression != null) {
|
||||
Object acl = this.objectAclExpression.getValue(this.evaluationContext, requestMessage);
|
||||
Assert.state(acl instanceof AccessControlList || acl instanceof CannedAccessControlList,
|
||||
"The 'objectAclExpression' ["
|
||||
+ this.objectAclExpression.getExpressionString()
|
||||
+ "] must evaluate to com.amazonaws.services.s3.model.AccessControlList " +
|
||||
"or must evaluate to com.amazonaws.services.s3.model.CannedAccessControlList. " +
|
||||
"Gotten: [" + acl + "]");
|
||||
|
||||
SetObjectAclRequest aclRequest;
|
||||
|
||||
if (acl instanceof AccessControlList) {
|
||||
aclRequest = new SetObjectAclRequest(bucketName, key, (AccessControlList) acl);
|
||||
}
|
||||
else {
|
||||
aclRequest = new SetObjectAclRequest(bucketName, key, (CannedAccessControlList) acl);
|
||||
}
|
||||
|
||||
final SetObjectAclRequest theAclRequest = aclRequest;
|
||||
progressListener = new S3ProgressListener() {
|
||||
|
||||
@Override
|
||||
public void onPersistableTransfer(PersistableTransfer persistableTransfer) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void progressChanged(ProgressEvent progressEvent) {
|
||||
if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(progressEvent.getEventType())) {
|
||||
transferManager.getAmazonS3Client().setObjectAcl(theAclRequest);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
if (this.s3ProgressListener != null) {
|
||||
progressListener = new S3ProgressListenerChain(this.s3ProgressListener, progressListener);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (progressListener != null) {
|
||||
return this.transferManager.upload(putObjectRequest, progressListener);
|
||||
}
|
||||
else {
|
||||
return this.transferManager.upload(putObjectRequest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Transfer download(Message<?> requestMessage) {
|
||||
Object payload = requestMessage.getPayload();
|
||||
Assert.state(payload instanceof File, "For the 'DOWNLOAD' operation the 'payload' must be of " +
|
||||
"'java.io.File' type, but gotten: [" + payload.getClass() + ']');
|
||||
|
||||
File targetFile = (File) payload;
|
||||
|
||||
String bucket = obtainBucket(requestMessage);
|
||||
|
||||
String key = null;
|
||||
if (this.keyExpression != null) {
|
||||
key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class);
|
||||
}
|
||||
else {
|
||||
key = targetFile.getName();
|
||||
}
|
||||
|
||||
Assert.state(key != null,
|
||||
"The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " +
|
||||
"Root object is: " + requestMessage);
|
||||
|
||||
if (targetFile.isDirectory()) {
|
||||
return this.transferManager.downloadDirectory(bucket, key, targetFile);
|
||||
}
|
||||
else {
|
||||
if (this.s3ProgressListener != null) {
|
||||
return this.transferManager.download(new GetObjectRequest(bucket, key), targetFile,
|
||||
this.s3ProgressListener);
|
||||
}
|
||||
else {
|
||||
return this.transferManager.download(bucket, key, targetFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Transfer copy(Message<?> requestMessage) {
|
||||
String sourceBucketName = obtainBucket(requestMessage);
|
||||
|
||||
String sourceKey = null;
|
||||
if (this.keyExpression != null) {
|
||||
sourceKey = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class);
|
||||
}
|
||||
|
||||
Assert.state(sourceKey != null,
|
||||
"The 'keyExpression' must not be null for 'copy' operation and can't evaluate to null. " +
|
||||
"Root object is: " + requestMessage);
|
||||
|
||||
String destinationBucketName = null;
|
||||
if (this.destinationBucketExpression != null) {
|
||||
destinationBucketName = this.destinationBucketExpression.getValue(this.evaluationContext, requestMessage,
|
||||
String.class);
|
||||
}
|
||||
|
||||
Assert.state(destinationBucketName != null,
|
||||
"The 'destinationBucketExpression' must not be null for 'copy' operation and can't evaluate to null. " +
|
||||
"Root object is: " + requestMessage);
|
||||
|
||||
String destinationKey = null;
|
||||
if (this.destinationKeyExpression != null) {
|
||||
destinationKey = this.destinationKeyExpression.getValue(this.evaluationContext, requestMessage,
|
||||
String.class);
|
||||
}
|
||||
|
||||
Assert.state(destinationKey != null,
|
||||
"The 'destinationKeyExpression' must not be null for 'copy' operation and can't evaluate to null. " +
|
||||
"Root object is: " + requestMessage);
|
||||
|
||||
|
||||
CopyObjectRequest copyObjectRequest =
|
||||
new CopyObjectRequest(sourceBucketName, sourceKey, destinationBucketName, destinationKey);
|
||||
return this.transferManager.copy(copyObjectRequest);
|
||||
}
|
||||
|
||||
private String obtainBucket(Message<?> requestMessage) {
|
||||
String bucketName;
|
||||
if (this.bucketExpression instanceof LiteralExpression) {
|
||||
bucketName = (String) this.bucketExpression.getValue();
|
||||
}
|
||||
else {
|
||||
bucketName = this.bucketExpression.getValue(this.evaluationContext, requestMessage, String.class);
|
||||
}
|
||||
Assert.state(bucketName != null, "The 'bucketExpression' ["
|
||||
+ this.bucketExpression.getExpressionString()
|
||||
+ "] must not evaluate to null. Root object is: " + requestMessage);
|
||||
|
||||
return bucketName;
|
||||
}
|
||||
|
||||
public enum Command {
|
||||
UPLOAD, DOWNLOAD, COPY
|
||||
}
|
||||
|
||||
/**
|
||||
* The callback to populate an {@link ObjectMetadata} for upload operation.
|
||||
* The message can be used as a metadata source.
|
||||
*/
|
||||
public interface UploadMetadataProvider {
|
||||
|
||||
void populateMetadata(ObjectMetadata metadata, Message<?> message);
|
||||
|
||||
}
|
||||
|
||||
private class MessageHeadersObjectMetadataProvider implements ObjectMetadataProvider {
|
||||
|
||||
private final MessageHeaders messageHeaders;
|
||||
|
||||
public MessageHeadersObjectMetadataProvider(MessageHeaders messageHeaders) {
|
||||
this.messageHeaders = messageHeaders;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void provideObjectMetadata(File file, ObjectMetadata metadata) {
|
||||
if (uploadMetadataProvider != null) {
|
||||
uploadMetadataProvider.populateMetadata(metadata,
|
||||
MessageBuilder.createMessage(file, this.messageHeaders));
|
||||
}
|
||||
if (metadata.getContentMD5() == null) {
|
||||
try {
|
||||
String contentMd5 = Md5Utils.md5AsBase64(file);
|
||||
metadata.setContentMD5(contentMd5);
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new AmazonClientException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.aws.outbound;
|
||||
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.http.client.methods.HttpRequestBase;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.expression.spel.standard.SpelExpressionParser;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.integration.annotation.ServiceActivator;
|
||||
import org.springframework.integration.channel.QueueChannel;
|
||||
import org.springframework.integration.config.EnableIntegration;
|
||||
import org.springframework.integration.expression.ValueExpression;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.MessageChannel;
|
||||
import org.springframework.messaging.MessageHandler;
|
||||
import org.springframework.messaging.PollableChannel;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
||||
import org.springframework.util.FileCopyUtils;
|
||||
|
||||
import com.amazonaws.event.ProgressEvent;
|
||||
import com.amazonaws.event.ProgressEventType;
|
||||
import com.amazonaws.event.ProgressListener;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.Headers;
|
||||
import com.amazonaws.services.s3.model.CannedAccessControlList;
|
||||
import com.amazonaws.services.s3.model.CopyObjectRequest;
|
||||
import com.amazonaws.services.s3.model.CopyObjectResult;
|
||||
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
|
||||
import com.amazonaws.services.s3.model.GetObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ListObjectsRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectListing;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.PutObjectResult;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.model.S3ObjectInputStream;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.amazonaws.services.s3.model.SetObjectAclRequest;
|
||||
import com.amazonaws.services.s3.transfer.Copy;
|
||||
import com.amazonaws.services.s3.transfer.PersistableTransfer;
|
||||
import com.amazonaws.services.s3.transfer.Transfer;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressListener;
|
||||
import com.amazonaws.services.s3.transfer.internal.S3ProgressPublisher;
|
||||
import com.amazonaws.util.BinaryUtils;
|
||||
import com.amazonaws.util.Md5Utils;
|
||||
import com.amazonaws.util.StringInputStream;
|
||||
import com.amazonaws.util.StringUtils;
|
||||
|
||||
/**
|
||||
* @author Artem Bilan
|
||||
*/
|
||||
@RunWith(SpringJUnit4ClassRunner.class)
|
||||
@ContextConfiguration
|
||||
@DirtiesContext
|
||||
public class S3MessageHandlerTests {
|
||||
|
||||
private static SpelExpressionParser PARSER = new SpelExpressionParser();
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder temporaryFolder = new TemporaryFolder();
|
||||
|
||||
@Autowired
|
||||
private AmazonS3 amazonS3;
|
||||
|
||||
@Autowired
|
||||
private MessageChannel s3SendChannel;
|
||||
|
||||
@Autowired
|
||||
private CountDownLatch transferCompletedLatch;
|
||||
|
||||
@Autowired
|
||||
private CountDownLatch aclLatch;
|
||||
|
||||
@Autowired
|
||||
private MessageChannel s3ProcessChannel;
|
||||
|
||||
@Autowired
|
||||
private PollableChannel s3ReplyChannel;
|
||||
|
||||
@Test
|
||||
public void testUploadFile() throws IOException, InterruptedException {
|
||||
File file = this.temporaryFolder.newFile("foo.mp3");
|
||||
Message<?> message = MessageBuilder.withPayload(file)
|
||||
.setHeader("s3Command", S3MessageHandler.Command.UPLOAD.name())
|
||||
.build();
|
||||
|
||||
this.s3SendChannel.send(message);
|
||||
|
||||
ArgumentCaptor<PutObjectRequest> putObjectRequestArgumentCaptor =
|
||||
ArgumentCaptor.forClass(PutObjectRequest.class);
|
||||
verify(this.amazonS3, atLeastOnce()).putObject(putObjectRequestArgumentCaptor.capture());
|
||||
|
||||
PutObjectRequest putObjectRequest = putObjectRequestArgumentCaptor.getValue();
|
||||
assertEquals("myBucket", putObjectRequest.getBucketName());
|
||||
assertEquals("foo.mp3", putObjectRequest.getKey());
|
||||
assertNull(putObjectRequest.getFile());
|
||||
assertNotNull(putObjectRequest.getInputStream());
|
||||
|
||||
ObjectMetadata metadata = putObjectRequest.getMetadata();
|
||||
assertEquals(Md5Utils.md5AsBase64(file), metadata.getContentMD5());
|
||||
assertEquals(0, metadata.getContentLength());
|
||||
assertEquals("audio/mpeg", metadata.getContentType());
|
||||
|
||||
ProgressListener listener = putObjectRequest.getGeneralProgressListener();
|
||||
S3ProgressPublisher.publishProgress(listener, ProgressEventType.TRANSFER_COMPLETED_EVENT);
|
||||
|
||||
assertTrue(this.transferCompletedLatch.await(10, TimeUnit.SECONDS));
|
||||
assertTrue(this.aclLatch.await(10, TimeUnit.SECONDS));
|
||||
|
||||
ArgumentCaptor<SetObjectAclRequest> setObjectAclRequestArgumentCaptor =
|
||||
ArgumentCaptor.forClass(SetObjectAclRequest.class);
|
||||
verify(this.amazonS3).setObjectAcl(setObjectAclRequestArgumentCaptor.capture());
|
||||
|
||||
SetObjectAclRequest setObjectAclRequest = setObjectAclRequestArgumentCaptor.getValue();
|
||||
|
||||
assertEquals("myBucket", setObjectAclRequest.getBucketName());
|
||||
assertEquals("foo.mp3", setObjectAclRequest.getKey());
|
||||
assertNull(setObjectAclRequest.getAcl());
|
||||
assertEquals(CannedAccessControlList.PublicReadWrite, setObjectAclRequest.getCannedAcl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUploadInputStream() throws IOException {
|
||||
InputStream payload = new StringInputStream("a");
|
||||
Message<?> message = MessageBuilder.withPayload(payload)
|
||||
.setHeader("s3Command", S3MessageHandler.Command.UPLOAD.name())
|
||||
.setHeader("key", "myStream")
|
||||
.build();
|
||||
|
||||
this.s3SendChannel.send(message);
|
||||
|
||||
ArgumentCaptor<PutObjectRequest> putObjectRequestArgumentCaptor =
|
||||
ArgumentCaptor.forClass(PutObjectRequest.class);
|
||||
verify(this.amazonS3, atLeastOnce()).putObject(putObjectRequestArgumentCaptor.capture());
|
||||
|
||||
PutObjectRequest putObjectRequest = putObjectRequestArgumentCaptor.getValue();
|
||||
assertEquals("myBucket", putObjectRequest.getBucketName());
|
||||
assertEquals("myStream", putObjectRequest.getKey());
|
||||
assertNull(putObjectRequest.getFile());
|
||||
assertNotNull(putObjectRequest.getInputStream());
|
||||
|
||||
ObjectMetadata metadata = putObjectRequest.getMetadata();
|
||||
assertEquals(Md5Utils.md5AsBase64(payload), metadata.getContentMD5());
|
||||
assertEquals(1, metadata.getContentLength());
|
||||
assertEquals(MediaType.APPLICATION_JSON_VALUE, metadata.getContentType());
|
||||
assertEquals("test.json", metadata.getContentDisposition());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDownloadDirectory() throws IOException {
|
||||
File directoryForDownload = this.temporaryFolder.newFolder("myFolder");
|
||||
Message<?> message = MessageBuilder.withPayload(directoryForDownload)
|
||||
.setHeader("s3Command", S3MessageHandler.Command.DOWNLOAD)
|
||||
.build();
|
||||
|
||||
this.s3SendChannel.send(message);
|
||||
|
||||
File[] fileArray = directoryForDownload.listFiles();
|
||||
assertNotNull(fileArray);
|
||||
assertEquals(2, fileArray.length);
|
||||
|
||||
List<File> files = Arrays.asList(fileArray);
|
||||
Collections.sort(files, new Comparator<File>() {
|
||||
|
||||
@Override
|
||||
public int compare(File o1, File o2) {
|
||||
return o1.getName().compareTo(o2.getName());
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
File file1 = files.get(0);
|
||||
assertEquals("bar", file1.getName());
|
||||
assertEquals("bb", FileCopyUtils.copyToString(new FileReader(file1)));
|
||||
|
||||
File file2 = files.get(1);
|
||||
assertEquals("foo", file2.getName());
|
||||
assertEquals("f", FileCopyUtils.copyToString(new FileReader(file2)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCopy() throws InterruptedException {
|
||||
Map<String, String> payload = new HashMap<>();
|
||||
payload.put("key", "mySource");
|
||||
payload.put("destination", "theirBucket");
|
||||
payload.put("destinationKey", "theirTarget");
|
||||
this.s3ProcessChannel.send(new GenericMessage<>(payload));
|
||||
|
||||
Message<?> receive = this.s3ReplyChannel.receive(10000);
|
||||
assertNotNull(receive);
|
||||
|
||||
Assert.assertThat(receive.getPayload(), instanceOf(Copy.class));
|
||||
Copy copy = (Copy) receive.getPayload();
|
||||
assertEquals("Copying object from myBucket/mySource to theirBucket/theirTarget", copy.getDescription());
|
||||
|
||||
copy.waitForCompletion();
|
||||
|
||||
assertEquals(Transfer.TransferState.Completed, copy.getState());
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@EnableIntegration
|
||||
public static class ContextConfiguration {
|
||||
|
||||
@Bean
|
||||
public AmazonS3 amazonS3() {
|
||||
AmazonS3 amazonS3 = mock(AmazonS3.class);
|
||||
|
||||
when(amazonS3.putObject(any(PutObjectRequest.class))).thenReturn(new PutObjectResult());
|
||||
when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class))).thenReturn(new ObjectMetadata());
|
||||
when(amazonS3.copyObject(any(CopyObjectRequest.class))).thenReturn(new CopyObjectResult());
|
||||
|
||||
ObjectListing objectListing = spy(new ObjectListing());
|
||||
|
||||
List<S3ObjectSummary> s3ObjectSummaries = new LinkedList<>();
|
||||
|
||||
S3ObjectSummary fileSummary1 = new S3ObjectSummary();
|
||||
fileSummary1.setBucketName("myBucket");
|
||||
fileSummary1.setKey("foo");
|
||||
fileSummary1.setSize(1);
|
||||
s3ObjectSummaries.add(fileSummary1);
|
||||
|
||||
S3ObjectSummary fileSummary2 = new S3ObjectSummary();
|
||||
fileSummary2.setBucketName("myBucket");
|
||||
fileSummary2.setKey("bar");
|
||||
fileSummary2.setSize(2);
|
||||
s3ObjectSummaries.add(fileSummary2);
|
||||
|
||||
when(objectListing.getObjectSummaries()).thenReturn(s3ObjectSummaries);
|
||||
when(amazonS3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
|
||||
|
||||
final S3Object file1 = new S3Object();
|
||||
file1.setBucketName("myBucket");
|
||||
file1.setKey("foo");
|
||||
try {
|
||||
byte[] data = "f".getBytes(StringUtils.UTF8);
|
||||
byte[] md5 = Md5Utils.computeMD5Hash(data);
|
||||
file1.getObjectMetadata().setHeader(Headers.ETAG, BinaryUtils.toHex(md5));
|
||||
S3ObjectInputStream content = new S3ObjectInputStream(new ByteArrayInputStream(data),
|
||||
mock(HttpRequestBase.class));
|
||||
file1.setObjectContent(content);
|
||||
}
|
||||
catch (Exception e) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
final S3Object file2 = new S3Object();
|
||||
file2.setBucketName("myBucket");
|
||||
file2.setKey("bar");
|
||||
try {
|
||||
byte[] data = "bb".getBytes(StringUtils.UTF8);
|
||||
byte[] md5 = Md5Utils.computeMD5Hash(data);
|
||||
file2.getObjectMetadata().setHeader(Headers.ETAG, BinaryUtils.toHex(md5));
|
||||
S3ObjectInputStream content = new S3ObjectInputStream(new ByteArrayInputStream(data),
|
||||
mock(HttpRequestBase.class));
|
||||
file2.setObjectContent(content);
|
||||
}
|
||||
catch (Exception e) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
doAnswer(new Answer<S3Object>() {
|
||||
|
||||
@Override
|
||||
public S3Object answer(InvocationOnMock invocation) throws Throwable {
|
||||
GetObjectRequest getObjectRequest = (GetObjectRequest) invocation.getArguments()[0];
|
||||
String key = getObjectRequest.getKey();
|
||||
if ("foo".equals(key)) {
|
||||
return file1;
|
||||
}
|
||||
else if ("bar".equals(key)) {
|
||||
return file2;
|
||||
}
|
||||
else {
|
||||
return (S3Object) invocation.callRealMethod();
|
||||
}
|
||||
}
|
||||
|
||||
}).when(amazonS3).getObject(any(GetObjectRequest.class));
|
||||
|
||||
doAnswer(new Answer<Object>() {
|
||||
|
||||
@Override
|
||||
public Object answer(InvocationOnMock invocation) throws Throwable {
|
||||
aclLatch().countDown();
|
||||
return null;
|
||||
}
|
||||
|
||||
}).when(amazonS3).setObjectAcl(any(SetObjectAclRequest.class));
|
||||
|
||||
return amazonS3;
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public CountDownLatch aclLatch() {
|
||||
return new CountDownLatch(1);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public CountDownLatch transferCompletedLatch() {
|
||||
return new CountDownLatch(1);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public S3ProgressListener s3ProgressListener() {
|
||||
return new S3ProgressListener() {
|
||||
|
||||
@Override
|
||||
public void onPersistableTransfer(PersistableTransfer persistableTransfer) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void progressChanged(ProgressEvent progressEvent) {
|
||||
if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(progressEvent.getEventType())) {
|
||||
transferCompletedLatch().countDown();
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ServiceActivator(inputChannel = "s3SendChannel")
|
||||
public MessageHandler s3MessageHandler() {
|
||||
S3MessageHandler s3MessageHandler = new S3MessageHandler(amazonS3(), "myBucket");
|
||||
s3MessageHandler.setCommandExpression(PARSER.parseExpression("headers.s3Command"));
|
||||
s3MessageHandler.setKeyExpression(PARSER.parseExpression("payload instanceof T(java.io.File) ? payload.name : headers.key"));
|
||||
s3MessageHandler.setObjectAclExpression(new ValueExpression<>(CannedAccessControlList.PublicReadWrite));
|
||||
s3MessageHandler.setUploadMetadataProvider(new S3MessageHandler.UploadMetadataProvider() {
|
||||
|
||||
@Override
|
||||
public void populateMetadata(ObjectMetadata metadata, Message<?> message) {
|
||||
if (message.getPayload() instanceof InputStream) {
|
||||
metadata.setContentLength(1);
|
||||
metadata.setContentType(MediaType.APPLICATION_JSON_VALUE);
|
||||
metadata.setContentDisposition("test.json");
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
s3MessageHandler.setS3ProgressListener(s3ProgressListener());
|
||||
return s3MessageHandler;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public PollableChannel s3ReplyChannel() {
|
||||
return new QueueChannel();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ServiceActivator(inputChannel = "s3ProcessChannel")
|
||||
public MessageHandler s3ProcessMessageHandler() {
|
||||
S3MessageHandler s3MessageHandler = new S3MessageHandler(amazonS3(), "myBucket", true);
|
||||
s3MessageHandler.setOutputChannel(s3ReplyChannel());
|
||||
s3MessageHandler.setCommand(S3MessageHandler.Command.COPY);
|
||||
s3MessageHandler.setKeyExpression(PARSER.parseExpression("payload.key"));
|
||||
s3MessageHandler.setDestinationBucketExpression(PARSER.parseExpression("payload.destination"));
|
||||
s3MessageHandler.setDestinationKeyExpression(PARSER.parseExpression("payload.destinationKey"));
|
||||
return s3MessageHandler;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user