Spring/spring-integration-aws
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-90: Rework locking logic in the KinesisMDChA

Browse Source 
Fixes https://github.com/spring-projects/spring-integration-aws/issues/90

* Add an internal `ShardConsumerManager` which is responsible to initiate
a locking for the shard key in the provided consumer group and populating
a `ShardConsumer` if `tryLock()` is successful or no `LockRegistry` at all
* Additional logic is added to always iterate over candidate shards if
`tryLock()` on the matter is not successful.
This way the current `KinesisMessageDrivenChannelAdapter` picks up
those shards which have been locked by the consumer which has just
left a cluster and unlocked distributed locks
* Now all the shards are considered as candidates independently of the
`streams` or `shardOffsets` configuration
* Improve Kinesis tests performance
This commit is contained in:
Artem Bilan
2018-07-16 13:17:13 -04:00
parent fc34f814e5
commit 0e02b0d470
4 changed files with 84 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/test/java/org/springframework/integration/aws/inbound/KinesisMessageDrivenChannelAdapterTests.java
View File

@@ -156,7 +156,7 @@ public class KinesisMessageDrivenChannelAdapterTests {
Map<?, ?> forLocking =
TestUtils.getPropertyValue(this.kinesisMessageDrivenChannelAdapter,
"shardLocksMonitor.forLocking", Map.class);
"shardConsumerManager.locks", Map.class);
Assert.assertThat(0, eventually(100, 100, equalsResult(forLocking::size)));

7
src/test/java/org/springframework/integration/aws/kinesis/KinesisIntegrationTests.java
View File

@@ -29,6 +29,7 @@ import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -170,6 +171,12 @@ public class KinesisIntegrationTests {
adapter.setCheckpointStore(checkpointStore());
adapter.setLockRegistry(lockRegistry());
adapter.setEmbeddedHeadersMapper(new EmbeddedJsonHeadersMessageMapper("foo"));
DirectFieldAccessor dfa = new DirectFieldAccessor(adapter);
dfa.setPropertyValue("describeStreamBackoff", 10);
dfa.setPropertyValue("consumerBackoff", 10);
dfa.setPropertyValue("idleBetweenPolls", 1);
return adapter;
}