KafkaItemWriter.write should not return until items are confirmed to have been written

Issue #3773
This commit is contained in:
Jacob Botuck
2021-01-05 11:35:08 -06:00
committed by Mahmoud Ben Hassine
parent a6f283b4a8
commit c886a60b4d
3 changed files with 31 additions and 5 deletions

View File

@@ -43,7 +43,9 @@ public abstract class KeyValueItemWriter<K, V> implements ItemWriter<V>, Initial
K key = itemKeyMapper.convert(item);
writeKeyValue(key, item);
}
flush();
}
protected void flush() throws Exception {}
/**
* Subclasses implement this method to write each item to key value store

View File

@@ -19,7 +19,12 @@ package org.springframework.batch.item.kafka;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.KeyValueItemWriter;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.util.Assert;
import org.springframework.util.concurrent.ListenableFuture;
import java.util.ArrayList;
import java.util.List;
/**
* <p>
@@ -34,16 +39,25 @@ import org.springframework.util.Assert;
public class KafkaItemWriter<K, T> extends KeyValueItemWriter<K, T> {
protected KafkaTemplate<K, T> kafkaTemplate;
private final List<ListenableFuture<SendResult<K, T>>> listenableFutures = new ArrayList<>();
@Override
protected void writeKeyValue(K key, T value) {
if (this.delete) {
this.kafkaTemplate.sendDefault(key, null);
listenableFutures.add(this.kafkaTemplate.sendDefault(key, null));
}
else {
this.kafkaTemplate.sendDefault(key, value);
listenableFutures.add(this.kafkaTemplate.sendDefault(key, value));
}
}
@Override
protected void flush() throws Exception{
kafkaTemplate.flush();
for(ListenableFuture<SendResult<K,T>> future: listenableFutures){
future.get();
}
listenableFutures.clear();
}
@Override
protected void init() {