From a70904ef08ff61ee4a72eedb0a2e8ee26a27f3f4 Mon Sep 17 00:00:00 2001 From: onobc Date: Fri, 11 Mar 2022 16:52:22 -0600 Subject: [PATCH] Add docker-compose for kafka cluster --- tools/kafka/docker-compose/README.adoc | 40 +++++++++++ .../docker-compose/control-center-ui.yml | 29 ++++++++ tools/kafka/docker-compose/kafka-cluster.yml | 71 +++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 tools/kafka/docker-compose/README.adoc create mode 100644 tools/kafka/docker-compose/control-center-ui.yml create mode 100644 tools/kafka/docker-compose/kafka-cluster.yml diff --git a/tools/kafka/docker-compose/README.adoc b/tools/kafka/docker-compose/README.adoc new file mode 100644 index 0000000..dbad27d --- /dev/null +++ b/tools/kafka/docker-compose/README.adoc @@ -0,0 +1,40 @@ +== Docker Compose scripts for Kafka + +=== Kafka Cluster +Runs a multi-node Kafka cluster w/ 3 brokers available at `locahost:9091,localhost:9092,localhost:9093`. + +To start the brokers run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml -d up +---- +To stop the brokers run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml down +---- + +TIP: To start the containers in the foreground simply add `-d` to the start command. To stop containers that are running in the foreground simple issue `CTRL-C` command. + + +=== Control Center UI +Runs a https://docs.confluent.io/platform/current/control-center/index.html[Confluent Control Center] that exposes a UI at http://locahost:9021. + +NOTE: It is expected that the brokers are already running + +To start the Control Center run the following command: +[source,shell] +---- +docker-compose -f ./control-center-ui.yml up +---- +To stop the Control Center run the following command: +[source,shell] +---- +docker-compose -f ./control-center-ui.yml down +---- + +TIP: The scripts can be chained together to start both cluster and UI in a single command +[source,shell] +---- +docker-compose -f kafka-cluster.yml -f ./control-center-ui.yml up` +---- \ No newline at end of file diff --git a/tools/kafka/docker-compose/control-center-ui.yml b/tools/kafka/docker-compose/control-center-ui.yml new file mode 100644 index 0000000..716be64 --- /dev/null +++ b/tools/kafka/docker-compose/control-center-ui.yml @@ -0,0 +1,29 @@ +version: '2' + +# Runs a Confluent Control Center UI instance on 'http://localhost:9021'. +# +# Pre-requisites: The brokers are already running. + +services: + + control-center: + image: confluentinc/cp-enterprise-control-center:7.0.1 + hostname: control-center + container_name: control-center + depends_on: + - broker1 + - broker2 + - broker3 + ports: + - "9021:9021" + environment: + CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker1:29091,broker2:29092,broker3:29093' + #CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083' + #CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088" + #CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088" + #CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + CONTROL_CENTER_REPLICATION_FACTOR: 1 + CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 + CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 + CONFLUENT_METRICS_TOPIC_REPLICATION: 1 + PORT: 9021 diff --git a/tools/kafka/docker-compose/kafka-cluster.yml b/tools/kafka/docker-compose/kafka-cluster.yml new file mode 100644 index 0000000..ee73406 --- /dev/null +++ b/tools/kafka/docker-compose/kafka-cluster.yml @@ -0,0 +1,71 @@ +version: '2' + +# Sets up a multi-node Kafka cluster w/ 3 brokers w/ the following listeners: +# - broker1 listening (external/internal): localhost:9091 / broker1:29091 +# - broker2 listening (external/internal): localhost:9092 / broker2:29092 +# - broker3 listening (external/internal): localhost:9093 / broker3:29093 + +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.0.1 + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker1: + image: confluentinc/cp-kafka:7.0.1 + hostname: broker1 + container_name: broker1 + depends_on: + - zookeeper + ports: + - "9091:9091" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker1:29091,EXTERNAL://localhost:9091 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_ADVERTISED_HOST_NAME: broker1 + + broker2: + image: confluentinc/cp-kafka:7.0.1 + hostname: broker2 + container_name: broker2 + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker2:29092,EXTERNAL://localhost:9092 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_ADVERTISED_HOST_NAME: broker2 + + broker3: + image: confluentinc/cp-kafka:7.0.1 + hostname: broker3 + container_name: broker3 + depends_on: + - zookeeper + ports: + - "9093:9093" + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker3:29093,EXTERNAL://localhost:9093 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_ADVERTISED_HOST_NAME: broker3