Add Kafka ItemWriter

This commit adds an ItemWriter implementation for Apache Kafka using
a KafkaTemplate with default topic set. Extending KeyValueItemWriter,
the writer requires a Converter to map the Kafka ProducerRecord's key
from the item.

Resolves BATCH-2765
This commit is contained in:
Mathieu Ouellet
2018-10-26 07:48:28 -04:00
committed by Mahmoud Ben Hassine
parent 96ec0d0d27
commit 06556dc97e
9 changed files with 379 additions and 0 deletions

View File

@@ -0,0 +1,57 @@
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.kafka;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.KeyValueItemWriter;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.util.Assert;
/**
* <p>
* An {@link ItemWriter} implementation for Apache Kafka using a
* {@link KafkaTemplate} with default topic configured.
* </p>
*
* @author Mathieu Ouellet
* @since 4.2
*
*/
public class KafkaItemWriter<K, T> extends KeyValueItemWriter<K, T> {
private KafkaTemplate<K, T> kafkaTemplate;
@Override
protected void writeKeyValue(K key, T value) {
if (this.delete) {
this.kafkaTemplate.sendDefault(key, null);
}
else {
this.kafkaTemplate.sendDefault(key, value);
}
}
@Override
protected void init() {
Assert.notNull(this.kafkaTemplate, "KafkaTemplate must not be null.");
Assert.notNull(this.kafkaTemplate.getDefaultTopic(), "KafkaTemplate must have the default topic set.");
}
public void setKafkaTemplate(KafkaTemplate<K, T> kafkaTemplate) {
this.kafkaTemplate = kafkaTemplate;
}
}

View File

@@ -0,0 +1,91 @@
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.item.kafka.builder;
import org.springframework.batch.item.kafka.KafkaItemWriter;
import org.springframework.core.convert.converter.Converter;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.util.Assert;
/**
* A builder implementation for the {@link KafkaItemWriter}
*
* @author Mathieu Ouellet
* @since 4.2
*/
public class KafkaItemWriterBuilder<K, V> {
private KafkaTemplate<K, V> kafkaTemplate;
private Converter<V, K> itemKeyMapper;
private boolean delete;
/**
* Establishes the KafkaTemplate to be used by the KafkaItemWriter.
* @param kafkaTemplate the template to be used
* @return this instance for method chaining
* @see KafkaItemWriter#setKafkaTemplate(KafkaTemplate)
*/
public KafkaItemWriterBuilder<K, V> kafkaTemplate(KafkaTemplate<K, V> kafkaTemplate) {
this.kafkaTemplate = kafkaTemplate;
return this;
}
/**
* Set the {@link Converter} to use to derive the key from the item.
*
* @param itemKeyMapper the Converter to use.
* @return The current instance of the builder.
* @see KafkaItemWriter#setItemKeyMapper(Converter)
*/
public KafkaItemWriterBuilder<K, V> itemKeyMapper(Converter<V, K> itemKeyMapper) {
this.itemKeyMapper = itemKeyMapper;
return this;
}
/**
* Indicates if the items being passed to the writer are all to be send as delete events to the topic. A delete
* event is made of a key with a null value. If set to false (default), the items will be send with provided value
* and key converter by the itemKeyMapper. If set to true, the items will be send with the key converter from the
* value by the itemKeyMapper and a null value.
*
* @param delete removal indicator.
* @return The current instance of the builder.
* @see KafkaItemWriter#setDelete(boolean)
*/
public KafkaItemWriterBuilder<K, V> delete(boolean delete) {
this.delete = delete;
return this;
}
/**
* Validates and builds a {@link KafkaItemWriter}.
*
* @return a {@link KafkaItemWriter}
*/
public KafkaItemWriter<K, V> build() {
Assert.notNull(this.kafkaTemplate, "kafkaTemplate is required.");
Assert.notNull(this.itemKeyMapper, "itemKeyMapper is required.");
KafkaItemWriter<K, V> writer = new KafkaItemWriter<>();
writer.setKafkaTemplate(this.kafkaTemplate);
writer.setItemKeyMapper(this.itemKeyMapper);
writer.setDelete(this.delete);
return writer;
}
}

View File

@@ -0,0 +1,25 @@
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Builders for Apache Kafka item reader and writer.
*
* @author Mathieu Ouellet
*/
@NonNullApi
package org.springframework.batch.item.kafka.builder;
import org.springframework.lang.NonNullApi;

View File

@@ -0,0 +1,25 @@
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Apache Kafka related readers and writers
*
* @author Mathieu Ouellet
*/
@NonNullApi
package org.springframework.batch.item.kafka;
import org.springframework.lang.NonNullApi;