diff --git a/tools/kafka/docker-compose/README.adoc b/tools/kafka/docker-compose/README.adoc new file mode 100644 index 000000000..5a6e50660 --- /dev/null +++ b/tools/kafka/docker-compose/README.adoc @@ -0,0 +1,64 @@ +== Docker Compose scripts for Kafka + +=== Kafka Cluster +Runs a multi-node Kafka cluster w/ 3 brokers available at `locahost:9091,localhost:9092,localhost:9093`. + +To start the brokers run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml up +---- +To stop the brokers run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml down +---- + +TIP: To start the containers in the foreground simply add `-d` to the start command. To stop containers that are running in the foreground simple issue `CTRL-C` command. + + +=== Control Center UI +Runs a https://docs.confluent.io/platform/current/control-center/index.html[Confluent Control Center] that exposes a UI at http://locahost:9021. + +NOTE: The scripts must be chained together as the UI depends on the brokers + +To start the brokers and the Control Center UI run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml -f ./control-center-ui.yml up +---- +To stop the brokers and the Control Center UI run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml -f ./control-center-ui.yml down +---- + +=== Schema Registry +Runs a https://docs.confluent.io/platform/current/schema-registry/index.html[Confluent Schema Registry] available at http://locahost:8081. + +NOTE: The scripts must be chained together as the schema registry depends on the brokers + +To start the brokers and the schema registry run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml -f ./schema-registry.yml up +---- +To stop the brokers and the schema registry run the following command: +[source,shell] +---- +docker-compose -f ./kafka-cluster.yml -f ./schema-registry.yml down +---- + +=== All The Things +The scripts can be chained together to start the cluster, UI and schema registry with the following commmand: +[source,shell] +---- +docker-compose -f kafka-cluster.yml -f ./control-center-ui.yml -f ./schema-registry.yml up +---- +Likewise, to stop all the things: +[source,shell] +---- +docker-compose -f kafka-cluster.yml -f ./control-center-ui.yml -f ./schema-registry.yml down +---- + +TIP: The only "UI" representation of the schema registry is by navigating to a `Topic` and then clicking the `Schema` tab for the topic \ No newline at end of file diff --git a/tools/kafka/docker-compose/control-center-ui.yml b/tools/kafka/docker-compose/control-center-ui.yml new file mode 100644 index 000000000..34cc5fb0b --- /dev/null +++ b/tools/kafka/docker-compose/control-center-ui.yml @@ -0,0 +1,29 @@ +version: '2' + +# Runs a Confluent Control Center UI instance on 'http://localhost:9021'. +# +# Pre-requisites: The brokers are already running. + +services: + + control-center: + image: confluentinc/cp-enterprise-control-center:7.0.1 + hostname: control-center + container_name: control-center + depends_on: + - broker1 + - broker2 + - broker3 + ports: + - "9021:9021" + environment: + CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker1:29091,broker2:29092,broker3:29093' + #CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083' + #CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088" + #CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088" + CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + CONTROL_CENTER_REPLICATION_FACTOR: 1 + CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 + CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 + CONFLUENT_METRICS_TOPIC_REPLICATION: 1 + PORT: 9021 diff --git a/tools/kafka/docker-compose/kafka-cluster.yml b/tools/kafka/docker-compose/kafka-cluster.yml new file mode 100644 index 000000000..ee7340630 --- /dev/null +++ b/tools/kafka/docker-compose/kafka-cluster.yml @@ -0,0 +1,71 @@ +version: '2' + +# Sets up a multi-node Kafka cluster w/ 3 brokers w/ the following listeners: +# - broker1 listening (external/internal): localhost:9091 / broker1:29091 +# - broker2 listening (external/internal): localhost:9092 / broker2:29092 +# - broker3 listening (external/internal): localhost:9093 / broker3:29093 + +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.0.1 + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker1: + image: confluentinc/cp-kafka:7.0.1 + hostname: broker1 + container_name: broker1 + depends_on: + - zookeeper + ports: + - "9091:9091" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker1:29091,EXTERNAL://localhost:9091 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_ADVERTISED_HOST_NAME: broker1 + + broker2: + image: confluentinc/cp-kafka:7.0.1 + hostname: broker2 + container_name: broker2 + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker2:29092,EXTERNAL://localhost:9092 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_ADVERTISED_HOST_NAME: broker2 + + broker3: + image: confluentinc/cp-kafka:7.0.1 + hostname: broker3 + container_name: broker3 + depends_on: + - zookeeper + ports: + - "9093:9093" + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker3:29093,EXTERNAL://localhost:9093 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_ADVERTISED_HOST_NAME: broker3 diff --git a/tools/kafka/docker-compose/schema-registry.yml b/tools/kafka/docker-compose/schema-registry.yml new file mode 100644 index 000000000..91a0dec66 --- /dev/null +++ b/tools/kafka/docker-compose/schema-registry.yml @@ -0,0 +1,22 @@ +version: '2' + +# Runs a Confluent Schema Registry instance on 'http://localhost:8081'. +# +# Pre-requisites: The brokers are already running. + +services: + schema-registry: + image: confluentinc/cp-schema-registry:7.0.1 + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker1 + - broker2 + - broker3 + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker1:29091,broker2:29092,broker3:29093' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: WARN