Add docker compose for Kafka cluster
This commit is contained in:
Chris Bono
64
tools/kafka/docker-compose/README.adoc
Normal file
64
tools/kafka/docker-compose/README.adoc
Normal file
@@ -0,0 +1,64 @@
|
||||
== Docker Compose scripts for Kafka
|
||||
|
||||
=== Kafka Cluster
|
||||
Runs a multi-node Kafka cluster w/ 3 brokers available at `locahost:9091,localhost:9092,localhost:9093`.
|
||||
|
||||
To start the brokers run the following command:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f ./kafka-cluster.yml up
|
||||
----
|
||||
To stop the brokers run the following command:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f ./kafka-cluster.yml down
|
||||
----
|
||||
|
||||
TIP: To start the containers in the foreground simply add `-d` to the start command. To stop containers that are running in the foreground simple issue `CTRL-C` command.
|
||||
|
||||
|
||||
=== Control Center UI
|
||||
Runs a https://docs.confluent.io/platform/current/control-center/index.html[Confluent Control Center] that exposes a UI at http://locahost:9021.
|
||||
|
||||
NOTE: The scripts must be chained together as the UI depends on the brokers
|
||||
|
||||
To start the brokers and the Control Center UI run the following command:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f ./kafka-cluster.yml -f ./control-center-ui.yml up
|
||||
----
|
||||
To stop the brokers and the Control Center UI run the following command:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f ./kafka-cluster.yml -f ./control-center-ui.yml down
|
||||
----
|
||||
|
||||
=== Schema Registry
|
||||
Runs a https://docs.confluent.io/platform/current/schema-registry/index.html[Confluent Schema Registry] available at http://locahost:8081.
|
||||
|
||||
NOTE: The scripts must be chained together as the schema registry depends on the brokers
|
||||
|
||||
To start the brokers and the schema registry run the following command:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f ./kafka-cluster.yml -f ./schema-registry.yml up
|
||||
----
|
||||
To stop the brokers and the schema registry run the following command:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f ./kafka-cluster.yml -f ./schema-registry.yml down
|
||||
----
|
||||
|
||||
=== All The Things
|
||||
The scripts can be chained together to start the cluster, UI and schema registry with the following commmand:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f kafka-cluster.yml -f ./control-center-ui.yml -f ./schema-registry.yml up
|
||||
----
|
||||
Likewise, to stop all the things:
|
||||
[source,shell]
|
||||
----
|
||||
docker-compose -f kafka-cluster.yml -f ./control-center-ui.yml -f ./schema-registry.yml down
|
||||
----
|
||||
|
||||
TIP: The only "UI" representation of the schema registry is by navigating to a `Topic` and then clicking the `Schema` tab for the topic
|
||||
29
tools/kafka/docker-compose/control-center-ui.yml
Normal file
29
tools/kafka/docker-compose/control-center-ui.yml
Normal file
@@ -0,0 +1,29 @@
|
||||
version: '2'
|
||||
|
||||
# Runs a Confluent Control Center UI instance on 'http://localhost:9021'.
|
||||
#
|
||||
# Pre-requisites: The brokers are already running.
|
||||
|
||||
services:
|
||||
|
||||
control-center:
|
||||
image: confluentinc/cp-enterprise-control-center:7.0.1
|
||||
hostname: control-center
|
||||
container_name: control-center
|
||||
depends_on:
|
||||
- broker1
|
||||
- broker2
|
||||
- broker3
|
||||
ports:
|
||||
- "9021:9021"
|
||||
environment:
|
||||
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker1:29091,broker2:29092,broker3:29093'
|
||||
#CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083'
|
||||
#CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088"
|
||||
#CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088"
|
||||
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
|
||||
CONTROL_CENTER_REPLICATION_FACTOR: 1
|
||||
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
|
||||
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
|
||||
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
|
||||
PORT: 9021
|
||||
71
tools/kafka/docker-compose/kafka-cluster.yml
Normal file
71
tools/kafka/docker-compose/kafka-cluster.yml
Normal file
@@ -0,0 +1,71 @@
|
||||
version: '2'
|
||||
|
||||
# Sets up a multi-node Kafka cluster w/ 3 brokers w/ the following listeners:
|
||||
# - broker1 listening (external/internal): localhost:9091 / broker1:29091
|
||||
# - broker2 listening (external/internal): localhost:9092 / broker2:29092
|
||||
# - broker3 listening (external/internal): localhost:9093 / broker3:29093
|
||||
|
||||
services:
|
||||
zookeeper:
|
||||
image: confluentinc/cp-zookeeper:7.0.1
|
||||
hostname: zookeeper
|
||||
container_name: zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOOKEEPER_CLIENT_PORT: 2181
|
||||
ZOOKEEPER_TICK_TIME: 2000
|
||||
|
||||
broker1:
|
||||
image: confluentinc/cp-kafka:7.0.1
|
||||
hostname: broker1
|
||||
container_name: broker1
|
||||
depends_on:
|
||||
- zookeeper
|
||||
ports:
|
||||
- "9091:9091"
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker1:29091,EXTERNAL://localhost:9091
|
||||
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
||||
KAFKA_ADVERTISED_HOST_NAME: broker1
|
||||
|
||||
broker2:
|
||||
image: confluentinc/cp-kafka:7.0.1
|
||||
hostname: broker2
|
||||
container_name: broker2
|
||||
depends_on:
|
||||
- zookeeper
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 2
|
||||
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker2:29092,EXTERNAL://localhost:9092
|
||||
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
||||
KAFKA_ADVERTISED_HOST_NAME: broker2
|
||||
|
||||
broker3:
|
||||
image: confluentinc/cp-kafka:7.0.1
|
||||
hostname: broker3
|
||||
container_name: broker3
|
||||
depends_on:
|
||||
- zookeeper
|
||||
ports:
|
||||
- "9093:9093"
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 3
|
||||
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker3:29093,EXTERNAL://localhost:9093
|
||||
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
|
||||
KAFKA_ADVERTISED_HOST_NAME: broker3
|
||||
22
tools/kafka/docker-compose/schema-registry.yml
Normal file
22
tools/kafka/docker-compose/schema-registry.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
version: '2'
|
||||
|
||||
# Runs a Confluent Schema Registry instance on 'http://localhost:8081'.
|
||||
#
|
||||
# Pre-requisites: The brokers are already running.
|
||||
|
||||
services:
|
||||
schema-registry:
|
||||
image: confluentinc/cp-schema-registry:7.0.1
|
||||
hostname: schema-registry
|
||||
container_name: schema-registry
|
||||
depends_on:
|
||||
- broker1
|
||||
- broker2
|
||||
- broker3
|
||||
ports:
|
||||
- "8081:8081"
|
||||
environment:
|
||||
SCHEMA_REGISTRY_HOST_NAME: schema-registry
|
||||
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker1:29091,broker2:29092,broker3:29093'
|
||||
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
|
||||
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: WARN
|
||||
Reference in New Issue
Block a user