Spring/spring-batch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ability to start reading from a custom offset in KafkaItemReader

Browse Source 
Issue #737
This commit is contained in:
Mahmoud Ben Hassine
2020-07-07 10:14:21 +02:00
parent 5fa821f25f
commit 15a393b8ef
2 changed files with 109 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
spring-batch-infrastructure/src/main/java/org/springframework/batch/item/kafka/KafkaItemReader.java
View File

@@ -140,12 +140,30 @@ public class KafkaItemReader<K, V> extends AbstractItemStreamItemReader<V> {
return this.saveState;
}
/**
* Setter for partition offsets. This mapping tells the reader the offset to start
* reading from in each partition. This is optional, defaults to starting from
* offset 0 in each partition. Passing an empty map makes the reader start
* from the offset stored in Kafka for the consumer group ID.
*
* <p><strong>In case of a restart, offsets stored in the execution context
* will take precedence.</strong></p>
*
* @param partitionOffsets mapping of starting offset in each partition
*/
public void setPartitionOffsets(Map<TopicPartition, Long> partitionOffsets) {
Assert.notNull(partitionOffsets, "partitionOffsets must not be null");
this.partitionOffsets = partitionOffsets;
}
@Override
public void open(ExecutionContext executionContext) {
this.kafkaConsumer = new KafkaConsumer<>(this.consumerProperties);
this.partitionOffsets = new HashMap<>();
for (TopicPartition topicPartition : this.topicPartitions) {
this.partitionOffsets.put(topicPartition, 0L);
if (this.partitionOffsets == null) {
this.partitionOffsets = new HashMap<>();
for (TopicPartition topicPartition : this.topicPartitions) {
this.partitionOffsets.put(topicPartition, 0L);
}
}
if (this.saveState && executionContext.containsKey(TOPIC_PARTITION_OFFSETS)) {
Map<TopicPartition, Long> offsets = (Map<TopicPartition, Long>) executionContext.get(TOPIC_PARTITION_OFFSETS);