From 0a000bdcd92f74676f3a24dc5de2b84c997efaa8 Mon Sep 17 00:00:00 2001 From: Newnius Date: Mon, 15 Apr 2019 20:24:13 +0800 Subject: [PATCH] update kafka, add 2.1.1, 2.2.0, update README --- kafka/2.1.0/README.md | 33 ++++++++++++++++++++++++++++- kafka/2.1.1/Dockerfile | 20 ++++++++++++++++++ kafka/2.1.1/README.md | 43 ++++++++++++++++++++++++++++++++++++++ kafka/2.1.1/start_kafka.sh | 21 +++++++++++++++++++ kafka/2.2.0/Dockerfile | 20 ++++++++++++++++++ kafka/2.2.0/README.md | 43 ++++++++++++++++++++++++++++++++++++++ kafka/2.2.0/start_kafka.sh | 21 +++++++++++++++++++ 7 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 kafka/2.1.1/Dockerfile create mode 100644 kafka/2.1.1/README.md create mode 100755 kafka/2.1.1/start_kafka.sh create mode 100644 kafka/2.2.0/Dockerfile create mode 100644 kafka/2.2.0/README.md create mode 100755 kafka/2.2.0/start_kafka.sh diff --git a/kafka/2.1.0/README.md b/kafka/2.1.0/README.md index 58776f2..5123867 100644 --- a/kafka/2.1.0/README.md +++ b/kafka/2.1.0/README.md @@ -1,4 +1,33 @@ -TBD +# deploy kafka with docker + +## docker swarm + +```bash +docker service create \ + --name kafka_1 \ + --hostname=kafka_1 \ + --network swarm-net \ + --replicas 1 \ + --detach=true \ + --env BROKER_ID=1 \ + --env ZOOKEEPER_CONNECT=zookeeper_node1:2181,zookeeper_node2:2181,zookeeper_node3:2181 \ + --endpoint-mode dnsrr \ + newnius/kafka:2.1.0 +``` + +## Configuration + +### By environment variable + +Currently, properties below are supported. + +| env | file | property | +| --- | --- | --- | +| BROKER_ID | server.properties | broker.id | +| ZOOKEEPER_CONNECT | server.properties | zookeeper.connect | + + +### By configuration file ```bash docker service create \ @@ -8,5 +37,7 @@ docker service create \ --detach=true \ --replicas 1 \ --endpoint-mode dnsrr \ + --mount type=bind,src=/data/kafka/,dst=/config/kafka/,readonly \ newnius/kafka:2.1.0 ``` + diff --git a/kafka/2.1.1/Dockerfile b/kafka/2.1.1/Dockerfile new file mode 100644 index 0000000..4c6841b --- /dev/null +++ b/kafka/2.1.1/Dockerfile @@ -0,0 +1,20 @@ +FROM openjdk:8-jre-alpine + +MAINTAINER Newnius + +# Prerequisites +RUN apk add --no-cache bash + +# Install Kafka +RUN wget -O kafka.tgz https://archive.apache.org/dist/kafka/2.1.1/kafka_2.12-2.1.1.tgz && \ + tar -xzf kafka.tgz -C /usr/local/ && rm kafka.tgz + +RUN ln -s /usr/local/kafka_2.12-2.1.1 /usr/local/kafka + +ENV KAFKA_HOME /usr/local/kafka + +WORKDIR $KAFKA_HOME + +ADD start_kafka.sh /etc/start_kafka.sh + +CMD ["/etc/start_kafka.sh"] diff --git a/kafka/2.1.1/README.md b/kafka/2.1.1/README.md new file mode 100644 index 0000000..d478f39 --- /dev/null +++ b/kafka/2.1.1/README.md @@ -0,0 +1,43 @@ +# deploy kafka with docker + +## docker swarm + +```bash +docker service create \ + --name kafka_1 \ + --hostname=kafka_1 \ + --network swarm-net \ + --replicas 1 \ + --detach=true \ + --env BROKER_ID=1 \ + --env ZOOKEEPER_CONNECT=zookeeper_node1:2181,zookeeper_node2:2181,zookeeper_node3:2181 \ + --endpoint-mode dnsrr \ + newnius/kafka:2.1.1 +``` + +## Configuration + +### By environment variable + +Currently, properties below are supported. + +| env | file | property | +| --- | --- | --- | +| BROKER_ID | server.properties | broker.id | +| ZOOKEEPER_CONNECT | server.properties | zookeeper.connect | + + +### By configuration file + +```bash +docker service create \ + --name kafka \ + --hostname=kafka \ + --network swarm-net \ + --detach=true \ + --replicas 1 \ + --endpoint-mode dnsrr \ + --mount type=bind,src=/data/kafka/,dst=/config/kafka/,readonly \ + newnius/kafka:2.1.1 +``` + diff --git a/kafka/2.1.1/start_kafka.sh b/kafka/2.1.1/start_kafka.sh new file mode 100755 index 0000000..e08e9b2 --- /dev/null +++ b/kafka/2.1.1/start_kafka.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +: ${EXTRA_CONF_DIR:=/config/kafka} + +## Overide default configuration files +if [ -d "$EXTRA_CONF_DIR" ]; then + cp $EXTRA_CONF_DIR/* /usr/local/kafka/config +fi + + +if ! [[ -z "${BROKER_ID}" ]]; then + sed -i '/broker.id=/d' /usr/local/kafka/config/server.properties + echo "broker.id=$BROKER_ID" >> /usr/local/kafka/config/server.properties +fi + +if ! [[ -z "${ZOOKEEPER_CONNECT}" ]]; then + sed -i '/zookeeper.connect=/d' /usr/local/kafka/config/server.properties + echo "zookeeper.connect=$ZOOKEEPER_CONNECT" >> /usr/local/kafka/config/server.properties +fi + +bash /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties diff --git a/kafka/2.2.0/Dockerfile b/kafka/2.2.0/Dockerfile new file mode 100644 index 0000000..0fe10e0 --- /dev/null +++ b/kafka/2.2.0/Dockerfile @@ -0,0 +1,20 @@ +FROM openjdk:8-jre-alpine + +MAINTAINER Newnius + +# Prerequisites +RUN apk add --no-cache bash + +# Install Kafka +RUN wget -O kafka.tgz https://archive.apache.org/dist/kafka/2.2.0/kafka_2.12-2.2.0.tgz && \ + tar -xzf kafka.tgz -C /usr/local/ && rm kafka.tgz + +RUN ln -s /usr/local/kafka_2.12-2.2.0 /usr/local/kafka + +ENV KAFKA_HOME /usr/local/kafka + +WORKDIR $KAFKA_HOME + +ADD start_kafka.sh /etc/start_kafka.sh + +CMD ["/etc/start_kafka.sh"] diff --git a/kafka/2.2.0/README.md b/kafka/2.2.0/README.md new file mode 100644 index 0000000..c3be614 --- /dev/null +++ b/kafka/2.2.0/README.md @@ -0,0 +1,43 @@ +# deploy kafka with docker + +## docker swarm + +```bash +docker service create \ + --name kafka_1 \ + --hostname=kafka_1 \ + --network swarm-net \ + --replicas 1 \ + --detach=true \ + --env BROKER_ID=1 \ + --env ZOOKEEPER_CONNECT=zookeeper_node1:2181,zookeeper_node2:2181,zookeeper_node3:2181 \ + --endpoint-mode dnsrr \ + newnius/kafka:2.2.0 +``` + +## Configuration + +### By environment variable + +Currently, properties below are supported. + +| env | file | property | +| --- | --- | --- | +| BROKER_ID | server.properties | broker.id | +| ZOOKEEPER_CONNECT | server.properties | zookeeper.connect | + + +### By configuration file + +```bash +docker service create \ + --name kafka \ + --hostname=kafka \ + --network swarm-net \ + --detach=true \ + --replicas 1 \ + --endpoint-mode dnsrr \ + --mount type=bind,src=/data/kafka/,dst=/config/kafka/,readonly \ + newnius/kafka:2.2.0 +``` + diff --git a/kafka/2.2.0/start_kafka.sh b/kafka/2.2.0/start_kafka.sh new file mode 100755 index 0000000..e08e9b2 --- /dev/null +++ b/kafka/2.2.0/start_kafka.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +: ${EXTRA_CONF_DIR:=/config/kafka} + +## Overide default configuration files +if [ -d "$EXTRA_CONF_DIR" ]; then + cp $EXTRA_CONF_DIR/* /usr/local/kafka/config +fi + + +if ! [[ -z "${BROKER_ID}" ]]; then + sed -i '/broker.id=/d' /usr/local/kafka/config/server.properties + echo "broker.id=$BROKER_ID" >> /usr/local/kafka/config/server.properties +fi + +if ! [[ -z "${ZOOKEEPER_CONNECT}" ]]; then + sed -i '/zookeeper.connect=/d' /usr/local/kafka/config/server.properties + echo "zookeeper.connect=$ZOOKEEPER_CONNECT" >> /usr/local/kafka/config/server.properties +fi + +bash /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties