From aea0e8f59110e1d3ae64affdcbdeff36a9640d60 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Fri, 26 Jan 2024 15:59:25 -0500 Subject: [PATCH] Kafka binder docs cleanup Fixing a few broken links in config options docs --- docs/modules/ROOT/nav.adoc | 1 + .../kafka/kafka-binder/config-options.adoc | 18 +++---- .../kafka/kafka-binder/dlq-partition.adoc | 46 ++++++++++++++++++ .../ROOT/pages/kafka/kafka-binder/dlq.adoc | 47 +------------------ .../kafka/kafka-binder/reset-offsets.adoc | 2 +- 5 files changed, 58 insertions(+), 56 deletions(-) create mode 100644 docs/modules/ROOT/pages/kafka/kafka-binder/dlq-partition.adoc diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 506378a87..1faa59365 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -62,6 +62,7 @@ **** xref:kafka/kafka-binder/admin-client-config-cust.adoc[] **** xref:kafka/kafka-binder/custom-health-ind.adoc[] **** xref:kafka/kafka-binder/dlq.adoc[] +**** xref:kafka/kafka-binder/dlq-partition.adoc[] **** xref:kafka/kafka-binder/partitions.adoc[] *** Reactive Kafka Binder **** xref:kafka/kafka-reactive-binder/overview.adoc[] diff --git a/docs/modules/ROOT/pages/kafka/kafka-binder/config-options.adoc b/docs/modules/ROOT/pages/kafka/kafka-binder/config-options.adoc index 1f3552ceb..7856b93e6 100644 --- a/docs/modules/ROOT/pages/kafka/kafka-binder/config-options.adoc +++ b/docs/modules/ROOT/pages/kafka/kafka-binder/config-options.adoc @@ -2,7 +2,7 @@ This section contains the configuration options used by the Apache Kafka binder. -For common configuration options and properties pertaining to the binder, see the https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/current/reference/html/spring-cloud-stream.html#binding-properties[binding properties] in core documentation. +For common configuration options and properties pertaining to the binder, see the https://docs.spring.io/spring-cloud-stream/docs/current/reference/html/spring-cloud-stream.html#binding-properties[binding properties] in core documentation. [[kafka-binder-properties]] == Kafka Binder Properties @@ -194,15 +194,15 @@ Note that this property is only applicable for pollable consumers. Default: not set. resetOffsets:: Whether to reset offsets on the consumer to the value provided by startOffset. -Must be false if a `KafkaBindingRebalanceListener` is provided; see <>. -See <> for more information about this property. +Must be false if a `KafkaBindingRebalanceListener` is provided; see xref:kafka/kafka-binder/rebalance_listener.adoc[rebalance listener] +See xref:kafka/kafka-binder/reset-offsets.adoc[reset-offsets] for more information about this property. + Default: `false`. startOffset:: The starting offset for new groups. Allowed values: `earliest` and `latest`. If the consumer group is set explicitly for the consumer 'binding' (through `spring.cloud.stream.bindings..group`), 'startOffset' is set to `earliest`. Otherwise, it is set to `latest` for the `anonymous` consumer group. -See <> for more information about this property. +See xref:kafka/kafka-binder/reset-offsets.adoc[reset-offsets] for more information about this property. + Default: null (equivalent to `earliest`). enableDlq:: @@ -210,17 +210,17 @@ When set to true, it enables DLQ behavior for the consumer. By default, messages that result in errors are forwarded to a topic named `error..`. The DLQ topic name can be configurable by setting the `dlqName` property or by defining a `@Bean` of type `DlqDestinationResolver`. This provides an alternative option to the more common Kafka replay scenario for the case when the number of errors is relatively small and replaying the entire original topic may be too cumbersome. -See <> processing for more information. +See xref:kafka/kafka-binder/dlq.adoc[kafka dlq processing] for more information. Starting with version 2.0, messages sent to the DLQ topic are enhanced with the following headers: `x-original-topic`, `x-exception-message`, and `x-exception-stacktrace` as `byte[]`. By default, a failed record is sent to the same partition number in the DLQ topic as the original record. -See <> for how to change that behavior. +See xref:kafka/kafka-binder/dlq-partition.adoc[dlq partition selection] for how to change that behavior. **Not allowed when `destinationIsPattern` is `true`.** + Default: `false`. dlqPartitions:: When `enableDlq` is true, and this property is not set, a dead letter topic with the same number of partitions as the primary topic(s) is created. Usually, dead-letter records are sent to the same partition in the dead-letter topic as the original record. -This behavior can be changed; see <>. +This behavior can be changed; see xref:kafka/kafka-binder/dlq-partition.adoc[dlq partition selection]. If this property is set to `1` and there is no `DqlPartitionFunction` bean, all dead-letter records will be written to partition `0`. If this property is greater than `1`, you **MUST** provide a `DlqPartitionFunction` bean. Note that the actual partition count is affected by the binder's `minPartitionCount` property. @@ -257,7 +257,7 @@ Default: `null` idleEventInterval:: The interval, in milliseconds, between events indicating that no messages have recently been received. Use an `ApplicationListener` to receive these events. -See <> for a usage example. +See xref:kafka/kafka-binder/pause_resume.adoc[pause-resume] for a usage example. + Default: `30000` destinationIsPattern:: @@ -393,7 +393,7 @@ The header contains a `RecordMetadata` object provided by the Kafka client; it i + `ResultMetadata meta = sendResultMsg.getHeaders().get(KafkaHeaders.RECORD_METADATA, RecordMetadata.class)` + -Failed sends go the producer error channel (if configured); see <>. +Failed sends go the producer error channel (if configured); see xref:kafka/kafka-binder/error-channels.adoc[Kafka error channels]. + Default: null. diff --git a/docs/modules/ROOT/pages/kafka/kafka-binder/dlq-partition.adoc b/docs/modules/ROOT/pages/kafka/kafka-binder/dlq-partition.adoc new file mode 100644 index 000000000..3bc866ef6 --- /dev/null +++ b/docs/modules/ROOT/pages/kafka/kafka-binder/dlq-partition.adoc @@ -0,0 +1,46 @@ +[[dlq-partition-selection]] +== Dead-Letter Topic Partition Selection + +By default, records are published to the Dead-Letter topic using the same partition as the original record. +This means the Dead-Letter topic must have at least as many partitions as the original record. + +To change this behavior, add a `DlqPartitionFunction` implementation as a `@Bean` to the application context. +Only one such bean can be present. +The function is provided with the consumer group, the failed `ConsumerRecord` and the exception. +For example, if you always want to route to partition 0, you might use: + +[source, java] +---- +@Bean +public DlqPartitionFunction partitionFunction() { + return (group, record, ex) -> 0; +} +---- +NOTE: If you set a consumer binding's `dlqPartitions` property to 1 (and the binder's `minPartitionCount` is equal to `1`), there is no need to supply a `DlqPartitionFunction`; the framework will always use partition 0. +If you set a consumer binding's `dlqPartitions` property to a value greater than `1` (or the binder's `minPartitionCount` is greater than `1`), you **must** provide a `DlqPartitionFunction` bean, even if the partition count is the same as the original topic's. + +It is also possible to define a custom name for the DLQ topic. +In order to do so, create an implementation of `DlqDestinationResolver` as a `@Bean` to the application context. +When the binder detects such a bean, that takes precedence, otherwise it will use the `dlqName` property. +If neither of these are found, it will default to `error..`. +Here is an example of `DlqDestinationResolver` as a `@Bean`. + +[source] +---- +@Bean +public DlqDestinationResolver dlqDestinationResolver() { + return (rec, ex) -> { + if (rec.topic().equals("word1")) { + return "topic1-dlq"; + } + else { + return "topic2-dlq"; + } + }; +} +---- + +One important thing to keep in mind when providing an implementation for `DlqDestinationResolver` is that the provisioner in the binder will not auto create topics for the application. +This is because there is no way for the binder to infer the names of all the DLQ topics the implementation might send to. +Therefore, if you provide DLQ names using this strategy, it is the application's responsibility to ensure that those topics are created beforehand. + diff --git a/docs/modules/ROOT/pages/kafka/kafka-binder/dlq.adoc b/docs/modules/ROOT/pages/kafka/kafka-binder/dlq.adoc index ce954df60..3c1065599 100644 --- a/docs/modules/ROOT/pages/kafka/kafka-binder/dlq.adoc +++ b/docs/modules/ROOT/pages/kafka/kafka-binder/dlq.adoc @@ -1,52 +1,6 @@ [[kafka-dlq-processing]] = Dead-Letter Topic Processing -[[dlq-partition-selection]] -== Dead-Letter Topic Partition Selection - -By default, records are published to the Dead-Letter topic using the same partition as the original record. -This means the Dead-Letter topic must have at least as many partitions as the original record. - -To change this behavior, add a `DlqPartitionFunction` implementation as a `@Bean` to the application context. -Only one such bean can be present. -The function is provided with the consumer group, the failed `ConsumerRecord` and the exception. -For example, if you always want to route to partition 0, you might use: - -[source, java] ----- -@Bean -public DlqPartitionFunction partitionFunction() { - return (group, record, ex) -> 0; -} ----- -NOTE: If you set a consumer binding's `dlqPartitions` property to 1 (and the binder's `minPartitionCount` is equal to `1`), there is no need to supply a `DlqPartitionFunction`; the framework will always use partition 0. -If you set a consumer binding's `dlqPartitions` property to a value greater than `1` (or the binder's `minPartitionCount` is greater than `1`), you **must** provide a `DlqPartitionFunction` bean, even if the partition count is the same as the original topic's. - -It is also possible to define a custom name for the DLQ topic. -In order to do so, create an implementation of `DlqDestinationResolver` as a `@Bean` to the application context. -When the binder detects such a bean, that takes precedence, otherwise it will use the `dlqName` property. -If neither of these are found, it will default to `error..`. -Here is an example of `DlqDestinationResolver` as a `@Bean`. - -[source] ----- -@Bean -public DlqDestinationResolver dlqDestinationResolver() { - return (rec, ex) -> { - if (rec.topic().equals("word1")) { - return "topic1-dlq"; - } - else { - return "topic2-dlq"; - } - }; -} ----- - -One important thing to keep in mind when providing an implementation for `DlqDestinationResolver` is that the provisioner in the binder will not auto create topics for the application. -This is because there is no way for the binder to infer the names of all the DLQ topics the implementation might send to. -Therefore, if you provide DLQ names using this strategy, it is the application's responsibility to ensure that those topics are created beforehand. - [[dlq-handling]] == Handling Records in a Dead-Letter Topic @@ -144,3 +98,4 @@ public class ReRouteDlqKApplication implements CommandLineRunner { } } ---- + diff --git a/docs/modules/ROOT/pages/kafka/kafka-binder/reset-offsets.adoc b/docs/modules/ROOT/pages/kafka/kafka-binder/reset-offsets.adoc index 5cd99c5c2..10287ee94 100644 --- a/docs/modules/ROOT/pages/kafka/kafka-binder/reset-offsets.adoc +++ b/docs/modules/ROOT/pages/kafka/kafka-binder/reset-offsets.adoc @@ -23,4 +23,4 @@ Set `resetOffsets` to `true` and `startOffset` to `latest`; the binding will per IMPORTANT: If a rebalance occurs after the initial assignment, the seeks will only be performed on any newly assigned partitions that were not assigned during the initial assignment. -For more control over topic offsets, see <>; when a listener is provided, `resetOffsets` should not be set to `true`, otherwise, that will cause an error. +For more control over topic offsets, see xref:kafka/kafka-binder/rebalance_listener.adoc[rebalance listener]; when a listener is provided, `resetOffsets` should not be set to `true`, otherwise, that will cause an error.