Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4095: Support Limiting (S)FTP Files Fetched

Browse Source 
JIRA: https://jira.spring.io/browse/INT-4095

Limit the number of remote files fetched on each poll (when it is necessary to fetch files).

Polishing - PR Comments

Polishing - Decouple MaxFetchSize from Poller

Polishing - PR Comments

Schemas and Docs

More Polishing

* Polishing according PR comments
This commit is contained in:
Gary Russell
2016-08-18 18:15:04 -04:00
committed by Artem Bilan
parent 53237aa833
commit ea4763faa9
25 changed files with 534 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
spring-integration-core/src/main/java/org/springframework/integration/endpoint/AbstractFetchLimitingMessageSource.java Normal file
View File

@@ -0,0 +1,56 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.endpoint;
/**
* A message source that can limit the number of remote objects it fetches.
*
* @author Gary Russell
* @since 5.0
*
*/
public abstract class AbstractFetchLimitingMessageSource<T> extends AbstractMessageSource<T>
implements MessageSourceManagement {
private volatile int maxFetchSize = Integer.MIN_VALUE;
@Override
public void setMaxFetchSize(int maxFetchSize) {
this.maxFetchSize = maxFetchSize;
}
@Override
public int getMaxFetchSize() {
return this.maxFetchSize;
}
@Override
protected Object doReceive() {
return doReceive(this.maxFetchSize);
}
/**
* Subclasses must implement this method. Typically the returned value will be the
* payload of type T, but the returned value may also be a Message instance whose
* payload is of type T.
* @param maxFetchSize the maximum number of messages to fetch if a fetch is
* necessary.
* @return The value returned.
*/
protected abstract Object doReceive(int maxFetchSize);
}

7
spring-integration-core/src/main/java/org/springframework/integration/endpoint/AbstractMessageSource.java
View File

@@ -128,10 +128,13 @@ public abstract class AbstractMessageSource<T> extends AbstractExpressionEvaluat
}
@Override
@SuppressWarnings("unchecked")
public final Message<T> receive() {
return buildMessage(doReceive());
}
@SuppressWarnings("unchecked")
protected Message<T> buildMessage(Object result) {
Message<T> message = null;
Object result = this.doReceive();
Map<String, Object> headers = this.evaluateHeaders();
if (result instanceof Message<?>) {
try {

51
spring-integration-core/src/main/java/org/springframework/integration/endpoint/MessageSourceManagement.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.endpoint;
import org.springframework.integration.support.management.IntegrationManagedResource;
import org.springframework.integration.support.management.MessageSourceMetrics;
import org.springframework.jmx.export.annotation.ManagedAttribute;
/**
* Message sources implementing this interface have additional properties that
* can be set or examined using JMX.
*
* @author Gary Russell
* @since 5.0
*
*/
@IntegrationManagedResource
public interface MessageSourceManagement extends MessageSourceMetrics {
/**
* Set the maximum number of objects the source should fetch if it is necessary to
* fetch objects. Setting the
* maxFetchSize to 0 disables remote fetching, a negative value indicates no limit.
* @param maxFetchSize the max fetch size; a negative value means unlimited.
*/
@ManagedAttribute(description = "Maximum objects to fetch")
void setMaxFetchSize(int maxFetchSize);
/**
* Return the max fetch size.
* @return the max fetch size.
* @see #setMaxFetchSize(int)
*/
@ManagedAttribute
int getMaxFetchSize();
}

45
spring-integration-core/src/main/java/org/springframework/integration/support/management/LifecycleMessageSourceManagement.java Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.support.management;
import org.springframework.context.Lifecycle;
import org.springframework.integration.endpoint.MessageSourceManagement;
/**
* An extension to {@link LifecycleMessageSourceMetrics} for sources that implement {@link MessageSourceManagement}.
*
* @author Gary Russell
* @since 5.0
*
*/
public class LifecycleMessageSourceManagement extends LifecycleMessageSourceMetrics implements MessageSourceManagement {
public LifecycleMessageSourceManagement(Lifecycle lifecycle, MessageSourceManagement delegate) {
super(lifecycle, delegate);
}
@Override
public void setMaxFetchSize(int maxFetchSize) {
((MessageSourceManagement) this.delegate).setMaxFetchSize(maxFetchSize);
}
@Override
public int getMaxFetchSize() {
return ((MessageSourceManagement) this.delegate).getMaxFetchSize();
}
}

2
spring-integration-core/src/main/java/org/springframework/integration/support/management/LifecycleMessageSourceMetrics.java
View File

@@ -33,7 +33,7 @@ public class LifecycleMessageSourceMetrics implements MessageSourceMetrics, Life
private final Lifecycle lifecycle;
private final MessageSourceMetrics delegate;
protected final MessageSourceMetrics delegate;
public LifecycleMessageSourceMetrics(Lifecycle lifecycle, MessageSourceMetrics delegate) {

48
spring-integration-core/src/main/java/org/springframework/integration/support/management/LifecycleTrackableMessageSourceManagement.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.support.management;
import org.springframework.context.Lifecycle;
import org.springframework.integration.endpoint.MessageSourceManagement;
/**
* An extension to {@link LifecycleTrackableMessageSourceMetrics} for sources
* that implement {@link MessageSourceManagement}.
*
* @author Gary Russell
* @since 5.0
*
*/
public class LifecycleTrackableMessageSourceManagement extends LifecycleTrackableMessageSourceMetrics
implements MessageSourceManagement {
public LifecycleTrackableMessageSourceManagement(Lifecycle lifecycle, MessageSourceManagement delegate) {
super(lifecycle, delegate);
}
@Override
public void setMaxFetchSize(int maxFetchSize) {
((MessageSourceManagement) this.delegate).setMaxFetchSize(maxFetchSize);
}
@Override
public int getMaxFetchSize() {
return ((MessageSourceManagement) this.delegate).getMaxFetchSize();
}
}

12
spring-integration-core/src/main/resources/org/springframework/integration/config/spring-integration-5.0.xsd
View File

@@ -5009,6 +5009,18 @@ See 'SmartLifecycleRoleController'.
</xsd:attribute>
</xsd:attributeGroup>
<xsd:attributeGroup name="maxFetchGroup">
<xsd:attribute name="max-fetch-size" type="xsd:string">
<xsd:annotation>
<xsd:documentation>
When fetching objects from some external resource, limit the number of such objects that
will be retrieved on each fetch. A negative value (default) indicates no limit; a value
of zero effectively disables fetching remote objects.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:attributeGroup>
<xsd:simpleType name="loggingLevel">
<xsd:restriction base="xsd:token">
<xsd:enumeration value="FATAL" />