Merge pull request #492 from olegz/INT-2470

This commit is contained in:
Gary Russell
2012-06-14 18:52:56 -04:00
11 changed files with 377 additions and 63 deletions

View File

@@ -0,0 +1,59 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.springframework.integration.util;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
/**
* The {@link LockRegistry} implementation which has no effect. Mainly used in cases where locking itself must be conditional
* but an extra IF statement would clutter the code.
* For example. In the FILE module FileWritingMessageHandler is initialized with this instance of LockRegistry by default
* since real locking is only required if its 'append' flag is set to true.
*
* @author Oleg Zhurakousky
* @since 2.2
*
*/
public final class PassThruLockRegistry implements LockRegistry {
public Lock obtain(Object lockKey) {
return new Lock() {
public void unlock() {
// noop
}
public boolean tryLock(long time, TimeUnit unit)
throws InterruptedException {
return true;
}
public boolean tryLock() {
return true;
}
public Condition newCondition() {
throw new UnsupportedOperationException("This method is not supported for this implementation of Lock");
}
public void lockInterruptibly() throws InterruptedException {
// noop
}
public void lock() {
// noop
}
};
}
}

View File

@@ -0,0 +1,65 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.util;
import java.io.IOException;
import java.util.concurrent.locks.Lock;
import org.springframework.integration.MessagingException;
/**
* A simple strategy callback class that allows you to provide
* a code that needs to be executed under {@link Lock} provided by
* {@link LockRegistry}
* A typical usage would be to provide implementation of {@link #whileLocked()} method and
* then call {@link #doWhileLocked()}
*
* @author Oleg Zhurakousky
* @since 2.2
*
*/
public abstract class WhileLockedProcessor {
private final Object key;
private final LockRegistry lockRegistry;
public WhileLockedProcessor(LockRegistry lockRegistry, Object key){
this.key = key;
this.lockRegistry = lockRegistry;
}
public final void doWhileLocked() throws IOException{
Lock lock = lockRegistry.obtain(key);
try {
lock.lockInterruptibly();
try {
this.whileLocked();
}
finally {
lock.unlock();
}
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new MessagingException("Thread was interrupted while performing task", e);
}
}
/**
* Override this method to provide the behavior that needs to be executed
* while under lock
* @throws IOException
*/
protected abstract void whileLocked() throws IOException;
}

View File

@@ -16,22 +16,28 @@
package org.springframework.integration.file;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.integration.Message;
import org.springframework.integration.MessageHandlingException;
import org.springframework.integration.core.MessageHandler;
import org.springframework.integration.handler.AbstractReplyProducingMessageHandler;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.util.DefaultLockRegistry;
import org.springframework.integration.util.LockRegistry;
import org.springframework.integration.util.PassThruLockRegistry;
import org.springframework.integration.util.WhileLockedProcessor;
import org.springframework.util.Assert;
import org.springframework.util.FileCopyUtils;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
/**
* A {@link MessageHandler} implementation that writes the Message payload to a
* file. If the payload is a File object, it will copy the File to the specified
@@ -49,7 +55,7 @@ import java.nio.charset.Charset;
* Likewise, any Object can be converted to a String based on its
* <code>toString()</code> method by the
* {@link org.springframework.integration.transformer.ObjectToStringTransformer}.
*
*
* @author Mark Fisher
* @author Iwein Fuld
* @author Alex Peters
@@ -60,6 +66,10 @@ public class FileWritingMessageHandler extends AbstractReplyProducingMessageHand
private volatile String temporaryFileSuffix =".writing";
private volatile boolean temporaryFileSuffixSet = false;
private volatile boolean append = false;
private final Log logger = LogFactory.getLog(this.getClass());
private volatile FileNameGenerator fileNameGenerator = new DefaultFileNameGenerator();
@@ -74,6 +84,8 @@ public class FileWritingMessageHandler extends AbstractReplyProducingMessageHand
private volatile boolean expectReply = true;
private volatile LockRegistry lockRegistry = new PassThruLockRegistry();
public FileWritingMessageHandler(File destinationDirectory) {
Assert.notNull(destinationDirectory, "Destination directory must not be null.");
this.destinationDirectory = destinationDirectory;
@@ -92,7 +104,27 @@ public class FileWritingMessageHandler extends AbstractReplyProducingMessageHand
}
public void setTemporaryFileSuffix(String temporaryFileSuffix) {
Assert.notNull(temporaryFileSuffix, "'temporaryFileSuffix' must not be null"); // empty string is OK
this.temporaryFileSuffix = temporaryFileSuffix;
this.temporaryFileSuffixSet = true;
}
/**
* Will set 'append' flag which will let his handler to append data to the
* existing file rather then creating a new file for each Message.
* If 'true' it will also create a real instance of the LockRegistry to ensure
* that there is no collisions when multiple threads are writing to the same file.
* Otherwise the LockRegistry is set to {@link PassThruLockRegistry} which has no effect.
*
* @param append
*/
public void setAppend(boolean append) {
this.append = append;
if (this.append){
this.lockRegistry = this.lockRegistry instanceof PassThruLockRegistry
? new DefaultLockRegistry()
: this.lockRegistry;
}
}
/**
@@ -148,6 +180,9 @@ public class FileWritingMessageHandler extends AbstractReplyProducingMessageHand
"Destination path [" + this.destinationDirectory + "] does not point to a directory.");
Assert.isTrue(this.destinationDirectory.canWrite(),
"Destination directory [" + this.destinationDirectory + "] is not writable.");
Assert.state(!(this.temporaryFileSuffixSet && this.append),
"'temporaryFileSuffix' can not be set when appending to an existing file");;
}
@Override
@@ -196,7 +231,7 @@ public class FileWritingMessageHandler extends AbstractReplyProducingMessageHand
/**
* Retrieves the File instance from the {@link FileHeaders#ORIGINAL_FILE}
* header if available. If the value is not a File instance or a String
* representation of a file path, this will return <code>null</code>.
* representation of a file path, this will return <code>null</code>.
*/
private File retrieveOriginalFileFromHeader(Message<?> message) {
Object value = message.getHeaders().get(FileHeaders.ORIGINAL_FILE);
@@ -209,63 +244,108 @@ public class FileWritingMessageHandler extends AbstractReplyProducingMessageHand
return null;
}
private File handleFileMessage(File sourceFile, File tempFile, File resultFile) throws IOException {
if (this.deleteSourceFiles) {
if (sourceFile.renameTo(resultFile)) {
return resultFile;
}
if (logger.isInfoEnabled()) {
logger.info(String.format("Failed to move file '%s'. Using copy and delete fallback.",
sourceFile.getAbsolutePath()));
}
private File handleFileMessage(final File sourceFile, File tempFile, final File resultFile) throws IOException {
if (this.append){
File fileToWriteTo = this.determineFileToWrite(resultFile, tempFile);
final FileOutputStream fos = new FileOutputStream(fileToWriteTo, this.append);
final FileInputStream fis = new FileInputStream(sourceFile);
WhileLockedProcessor whileLockedProcessor = new WhileLockedProcessor(this.lockRegistry, fileToWriteTo.getAbsolutePath()){
@Override
protected void whileLocked() throws IOException {
FileCopyUtils.copy(fis, fos);
}
};
whileLockedProcessor.doWhileLocked();
this.cleanUpAfterCopy(fileToWriteTo, resultFile, sourceFile);
return resultFile;
}
FileCopyUtils.copy(sourceFile, tempFile);
this.renameTo(tempFile, resultFile);
if (this.deleteSourceFiles) {
sourceFile.delete();
else {
if (this.deleteSourceFiles) {
if (sourceFile.renameTo(resultFile)) {
return resultFile;
}
if (logger.isInfoEnabled()) {
logger.info(String.format("Failed to move file '%s'. Using copy and delete fallback.",
sourceFile.getAbsolutePath()));
}
}
FileCopyUtils.copy(sourceFile, tempFile);
this.cleanUpAfterCopy(tempFile, resultFile, sourceFile);
return resultFile;
}
}
private File handleByteArrayMessage(final byte[] bytes, File originalFile, File tempFile, final File resultFile) throws IOException {
File fileToWriteTo = this.determineFileToWrite(resultFile, tempFile);
final FileOutputStream fos = new FileOutputStream(fileToWriteTo, this.append);
WhileLockedProcessor whileLockedProcessor = new WhileLockedProcessor(this.lockRegistry, fileToWriteTo.getAbsolutePath()){
@Override
protected void whileLocked() throws IOException {
FileCopyUtils.copy(bytes, fos);
}
};
whileLockedProcessor.doWhileLocked();
this.cleanUpAfterCopy(fileToWriteTo, resultFile, originalFile);
return resultFile;
}
private File handleByteArrayMessage(byte[] bytes, File originalFile, File tempFile, File resultFile) throws IOException {
FileCopyUtils.copy(bytes, tempFile);
this.renameTo(tempFile, resultFile);
if (this.deleteSourceFiles && originalFile != null) {
originalFile.delete();
}
private File handleStringMessage(final String content, File originalFile, File tempFile, final File resultFile) throws IOException {
File fileToWriteTo = this.determineFileToWrite(resultFile, tempFile);
final OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(fileToWriteTo, this.append), this.charset);
WhileLockedProcessor whileLockedProcessor = new WhileLockedProcessor(this.lockRegistry, fileToWriteTo.getAbsolutePath()){
@Override
protected void whileLocked() throws IOException {
FileCopyUtils.copy(content, writer);
}
};
whileLockedProcessor.doWhileLocked();
this.cleanUpAfterCopy(fileToWriteTo, resultFile, originalFile);
return resultFile;
}
private File handleStringMessage(String content, File originalFile, File tempFile, File resultFile) throws IOException {
OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(tempFile), this.charset);
FileCopyUtils.copy(content, writer);
this.renameTo(tempFile, resultFile);
private File determineFileToWrite(File resultFile, File tempFile){
File fileToWriteTo = null;
if (this.append){
fileToWriteTo = resultFile;
}
else {
fileToWriteTo = tempFile;
}
return fileToWriteTo;
}
private void cleanUpAfterCopy(File fileToWriteTo, File resultFile, File originalFile) throws IOException{
if (!this.append){
this.renameTo(fileToWriteTo, resultFile);
}
if (this.deleteSourceFiles && originalFile != null) {
originalFile.delete();
}
return resultFile;
}
private void renameTo(File tempFile, File resultFile) throws IOException{
Assert.notNull(resultFile, "'resultFile' must not be null");
Assert.notNull(tempFile, "'tempFile' must not be null");
if (resultFile.exists()) {
if (resultFile.setWritable(true, false) && resultFile.delete()){
if (resultFile.setWritable(true, false) && resultFile.delete()){
if (!tempFile.renameTo(resultFile)) {
throw new IOException("Failed to rename file '" + tempFile.getAbsolutePath() + "' to '" + resultFile.getAbsolutePath() + "'");
}
}
else {
throw new IOException("Failed to rename file '" + tempFile.getAbsolutePath() + "' to '" + resultFile.getAbsolutePath() +
throw new IOException("Failed to rename file '" + tempFile.getAbsolutePath() + "' to '" + resultFile.getAbsolutePath() +
"' since '" + resultFile.getName() + "' is not writable or can not be deleted");
}
}
else {
else {
if (!tempFile.renameTo(resultFile)) {
throw new IOException("Failed to rename file '" + tempFile.getAbsolutePath() + "' to '" + resultFile.getAbsolutePath() + "'");
}
}
}
}

View File

@@ -16,19 +16,19 @@
package org.springframework.integration.file.config;
import org.springframework.integration.file.DefaultFileNameGenerator;
import org.w3c.dom.Element;
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.xml.ParserContext;
import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
import org.springframework.integration.file.DefaultFileNameGenerator;
import org.springframework.util.StringUtils;
/**
* A common helper class for the 'outbound-channel-adapter' and 'outbound-gateway'
* element parsers. Both of those are responsible for creating an instance of
* {@link org.springframework.integration.file.FileWritingMessageHandler}.
*
*
* @author Mark Fisher
* @author Artem Bilan
* @since 1.0.3
@@ -47,6 +47,7 @@ abstract class FileWritingMessageHandlerBeanDefinitionBuilder {
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "auto-create-directory");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "delete-source-files");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "temporary-file-suffix");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "append");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "charset");
String remoteFileNameGenerator = element.getAttribute("filename-generator");
String remoteFileNameGeneratorExpression = element.getAttribute("filename-generator-expression");

View File

@@ -33,25 +33,31 @@ import org.springframework.integration.file.FileWritingMessageHandler;
* @since 1.0.3
*/
public class FileWritingMessageHandlerFactoryBean extends AbstractSimpleMessageHandlerFactoryBean<FileWritingMessageHandler>{
private volatile File directory;
private volatile String charset;
private volatile FileNameGenerator fileNameGenerator;
private volatile Boolean deleteSourceFiles;
private volatile Boolean autoCreateDirectory;
private volatile Boolean requiresReply;
private volatile Long sendTimeout;
private volatile String temporaryFileSuffix;
private volatile boolean append;
private volatile boolean expectReply = true;
public void setAppend(boolean append) {
this.append = append;
}
public void setDirectory(File directory) {
this.directory = directory;
}
@@ -113,6 +119,7 @@ public class FileWritingMessageHandlerFactoryBean extends AbstractSimpleMessageH
handler.setTemporaryFileSuffix(this.temporaryFileSuffix);
}
handler.setExpectReply(this.expectReply);
handler.setAppend(this.append);
return handler;
}
}

View File

@@ -272,6 +272,19 @@ Only files matching this regular expression will be picked up by this adapter.
<xsd:annotation>
<xsd:documentation>
Extension used when uploading files. We change it after we know it's uploaded.
This attribute is mutualy exclusive with 'append' since the append is done to the
actual file and not its temporary counterpart. The default value of this attribute (i.e., .writing)
is ignored when 'append' is set to true.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="append" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
Will append to the existing file if such file exists otherwise the new file will be created as usual
but once created the subsequent data will be appended to it.
This attribute is mutualy exclusive with the 'temporary-file-suffix' since append is done to the
actual file and not its temporary counterpart.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>

View File

@@ -16,11 +16,18 @@
package org.springframework.integration.file;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.Message;
import org.springframework.integration.MessageChannel;
@@ -29,12 +36,6 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.FileCopyUtils;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
import static org.junit.Assert.*;
/**
* //INT-2275
*

View File

@@ -4,14 +4,12 @@
xmlns:si="http://www.springframework.org/schema/integration"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:file="http://www.springframework.org/schema/integration/file"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/integration
http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/file
http://www.springframework.org/schema/integration/file/spring-integration-file.xsd">
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/file http://www.springframework.org/schema/integration/file/spring-integration-file.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.1.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
<si:channel id="testChannel"/>
@@ -41,9 +39,24 @@
order="555"
auto-startup="false"
directory="${java.io.tmpdir}"/>
<file:outbound-channel-adapter id="usageChannel"
filename-generator-expression="'fileToAppend.txt'"
append="true"
directory="test"/>
<si:channel id="usageChannelConcurrent">
<si:dispatcher task-executor="executor"/>
</si:channel>
<file:outbound-channel-adapter channel="usageChannelConcurrent"
filename-generator-expression="'fileToAppendConcurrent.txt'"
append="true"
directory="test"/>
<bean id="customFileNameGenerator" class="org.springframework.integration.file.config.CustomFileNameGenerator"/>
<context:property-placeholder/>
<task:executor id="executor" pool-size="100"/>
</beans>

View File

@@ -30,18 +30,22 @@ import org.junit.runner.RunWith;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.MessageChannel;
import org.springframework.integration.endpoint.EventDrivenConsumer;
import org.springframework.integration.file.DefaultFileNameGenerator;
import org.springframework.integration.file.FileWritingMessageHandler;
import org.springframework.integration.message.GenericMessage;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.FileCopyUtils;
/**
* @author Mark Fisher
* @author Marius Bogoevici
* @author Iwein Fuld
* @author Gary Russell
* @author Oleg Zhurakousky
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
@@ -62,6 +66,12 @@ public class FileOutboundChannelAdapterParserTests {
@Autowired
EventDrivenConsumer adapterWithCharset;
@Autowired
MessageChannel usageChannel;
@Autowired
MessageChannel usageChannelConcurrent;
@Test
public void simpleAdapter() {
DirectFieldAccessor adapterAccessor = new DirectFieldAccessor(simpleAdapter);
@@ -126,4 +136,61 @@ public class FileOutboundChannelAdapterParserTests {
assertEquals(Charset.forName("UTF-8"), handlerAccessor.getPropertyValue("charset"));
}
@Test
public void adapterUsageWithAppend() throws Exception{
String expectedFileContent = "Initial File Content:String content:byte[] content:File content";
File testFile = new File("test/fileToAppend.txt");
if (testFile.exists()){
testFile.delete();
}
usageChannel.send(new GenericMessage<String>("Initial File Content:"));
usageChannel.send(new GenericMessage<String>("String content:"));
usageChannel.send(new GenericMessage<byte[]>("byte[] content:".getBytes()));
usageChannel.send(new GenericMessage<File>(new File("test/input.txt")));
String actualFileContent = new String(FileCopyUtils.copyToByteArray(testFile));
assertEquals(expectedFileContent, actualFileContent);
}
@Test
public void adapterUsageWithAppendConcurrent() throws Exception{
File testFile = new File("test/fileToAppendConcurrent.txt");
if (testFile.exists()){
testFile.delete();
}
StringBuffer aBuffer = new StringBuffer();
StringBuffer bBuffer = new StringBuffer();
for (int i = 0; i < 100000; i++) {
aBuffer.append("a");
bBuffer.append("b");
}
String aString = aBuffer.toString();
String bString = bBuffer.toString();
for (int i = 0; i < 1; i ++) {
usageChannelConcurrent.send(new GenericMessage<String>(aString));
usageChannelConcurrent.send(new GenericMessage<String>(bString));
}
Thread.sleep(2000);
String actualFileContent = new String(FileCopyUtils.copyToByteArray(testFile));
int beginningIndex = 0;
for (int i = 0; i < 2; i++) {
assertAllCharactersAreSame(actualFileContent.substring(beginningIndex, beginningIndex+99999));
beginningIndex += 100000;
}
}
private void assertAllCharactersAreSame(String substring){
char[] characters = substring.toCharArray();
char c = characters[0];
for (char character : characters) {
assertEquals(c, character);
}
}
}

View File

@@ -0,0 +1 @@
File content

View File

@@ -206,6 +206,13 @@
to be converted to file content you could extend the FileWritingMessageHandler, but a much
better option is to rely on a <classname>Transformer</classname>.
</para>
<para>
Since Spring Integration version 2.2. you can append Message content to the existing file instead of creating a new
File each time. To do so set <emphasis>append</emphasis> attribute to <code></code>;
Note that this attribute is mutually exclusive with <emphasis>temporary-file-suffix</emphasis> attribute since when
appending content to the existing file the adapter no longer uses temporary file.
</para>
</section>
<section id="file-transforming">