kafka в docker swarm

Thank you for reading this post, don't forget to subscribe!

Стек для запуска kafka в docker swarm выглядит следующим образом:

[codesyntax lang="bash" blockstate="collapsed"]

version: '3.4'

volumes:
  kafka-1:
    driver: "rexray/rbd:latest"
    driver_opts:
       size: 1
  kafka-2:
    driver: "rexray/rbd:latest"
    driver_opts:
       size: 1
  kafka-3:
    driver: "rexray/rbd:latest"
    driver_opts:
       size: 1
  zk-1:
    driver: "rexray/rbd:latest"
    driver_opts:
       size: 2
  zk-2:
    driver: "rexray/rbd:latest"
    driver_opts:
       size: 2
  zk-3:
    driver: "rexray/rbd:latest"
    driver_opts:
       size: 2

services:
  zk-1:
    image: confluentinc/cp-zookeeper:4.0.0
    hostname: zookeeper-1
    networks:
      - kafka_default
      - proxy
    environment:
      ZOOKEEPER_HEAP_SIZE: "-Xmx256m -Xms256m"
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: 0.0.0.0:2888:3888;zk-2:2888:3888;zk-3:2888:3888
    ports:
      - 2181:2181
    volumes:
      - zk-1:/var/lib/zookeeper/data
    deploy:
      mode: replicated
      replicas: 1
      labels:
       - com.df.notify=true
       - com.df.serviceDomain=zk1.test.ru
       - com.df.port=2181
  zk-2:
    image: confluentinc/cp-zookeeper:4.0.0
    hostname: zookeeper-2
    networks:
      - kafka_default
      - proxy
    environment:
      ZOOKEEPER_HEAP_SIZE: "-Xmx256m -Xms256m"
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_SERVERS: zk-1:2888:3888;0.0.0.0:2888:3888;zk-3:2888:3888
    ports:
      - 2182:2181
    volumes:
      - zk-2:/var/lib/zookeeper/data
    deploy:
      mode: replicated
      replicas: 1
      labels:
       - com.df.notify=true
       - com.df.serviceDomain=zk2.test.ru
       - com.df.port=2181
  zk-3:
    image: confluentinc/cp-zookeeper:4.0.0
    hostname: zookeeper-3
    networks:
      - kafka_default
      - proxy
    environment:
      ZOOKEEPER_HEAP_SIZE: "-Xmx256m -Xms256m"
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_SERVERS: zk-1:2888:3888;zk-2:2888:3888;0.0.0.0:2888:3888
    ports:
      - 2183:2181
    volumes:
      - zk-3:/var/lib/zookeeper/data
    deploy:
      mode: replicated
      replicas: 1
      labels:
       - com.df.notify=true
       - com.df.serviceDomain=zk3.test.ru
       - com.df.port=2181

  kafka-1:
    image: confluentinc/cp-kafka:4.0.0
    hostname: kafka-1
    networks:
      - kafka_default
    environment:
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
      KAFKA_ZOOKEEPER_CONNECT: zk-1:2181,zk-2:2181,zk-3:2181
      KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:9192
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_JMX_PORT: 9181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9192,EXTERNAL://kafka.test.ru:9092
      KAFKA_JMX_HOSTNAME: kafka-1
    ports:
      - 9092:9092
    volumes:
      - kafka-1:/var/lib/kafka/data
    deploy:
      mode: replicated
      replicas: 1

  kafka-2:
    image: confluentinc/cp-kafka:4.0.0
    hostname: kafka-2
    networks:
      - kafka_default
      - proxy
    environment:
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
      KAFKA_ZOOKEEPER_CONNECT: zk-1:2181,zk-2:2181,zk-3:2181
      KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:9192
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_JMX_PORT: 9181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:9192,EXTERNAL://kafka.test.ru:9093
      KAFKA_JMX_HOSTNAME: kafka-2
    ports:
      - 9093:9092
    volumes:
      - kafka-2:/var/lib/kafka/data
    deploy:
      mode: replicated
      replicas: 1

  kafka-3:
    image: confluentinc/cp-kafka:4.0.0
    hostname: kafka-3
    networks:
      - kafka_default
      - proxy
    environment:
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
      KAFKA_ZOOKEEPER_CONNECT: zk-1:2181,zk-2:2181,zk-3:2181
      KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:9192
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_JMX_PORT: 9181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:9192,EXTERNAL://kafka.test.ru:9094
      KAFKA_JMX_HOSTNAME: kafka-3
    ports:
      - 9094:9092
    volumes:
      - kafka-3:/var/lib/kafka/data
    deploy:
      mode: replicated
      replicas: 1

  kafka-manager:
    image: sheepkiller/kafka-manager
    hostname: kafka-manager
    networks:
      - kafka_default
      - proxy
    environment:
      ZK_HOSTS: zk-1:2181,zk-2:2181,zk-3:2181
      JMX_PORT: 9181
      APPLICATION_SECRET: letmein
    ports:
      - '9000'
    deploy:
      mode: replicated
      replicas: 1
      labels:
       - com.df.notify=true
       - com.df.serviceDomain=kafkamgr.test.ru
       - com.df.port=9000
networks:
  kafka_default:
    driver: overlay
  proxy:
    external: true

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

version: '3.4'

volumes:

kafka-1:

driver: "rexray/rbd:latest"

driver_opts:

size: 1

kafka-2:

driver: "rexray/rbd:latest"

driver_opts:

size: 1

kafka-3:

driver: "rexray/rbd:latest"

driver_opts:

size: 1

zk-1:

driver: "rexray/rbd:latest"

driver_opts:

size: 2

zk-2:

driver: "rexray/rbd:latest"

driver_opts:

size: 2

zk-3:

driver: "rexray/rbd:latest"

driver_opts:

size: 2

services:

zk-1:

image: confluentinc/cp-zookeeper:4.0.0

hostname: zookeeper-1

networks:

- kafka_default

- proxy

environment:

ZOOKEEPER_HEAP_SIZE: "-Xmx256m -Xms256m"

KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"

ZOOKEEPER_CLIENT_PORT: 2181

ZOOKEEPER_SERVER_ID: 1

ZOOKEEPER_SERVERS: 0.0.0.0:2888:3888;zk-2:2888:3888;zk-3:2888:3888

ports:

- 2181:2181

volumes:

- zk-1:/var/lib/zookeeper/data

deploy:

mode: replicated

replicas: 1

labels:

- com.df.notify=true

- com.df.serviceDomain=zk1.test.ru

- com.df.port=2181

zk-2:

image: confluentinc/cp-zookeeper:4.0.0

hostname: zookeeper-2

networks:

- kafka_default

- proxy

environment:

ZOOKEEPER_HEAP_SIZE: "-Xmx256m -Xms256m"

KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"

ZOOKEEPER_CLIENT_PORT: 2181

ZOOKEEPER_SERVER_ID: 2

ZOOKEEPER_SERVERS: zk-1:2888:3888;0.0.0.0:2888:3888;zk-3:2888:3888

ports:

- 2182:2181

volumes:

- zk-2:/var/lib/zookeeper/data

deploy:

mode: replicated

replicas: 1

labels:

- com.df.notify=true

- com.df.serviceDomain=zk2.test.ru

- com.df.port=2181

zk-3:

image: confluentinc/cp-zookeeper:4.0.0

hostname: zookeeper-3

networks:

- kafka_default

- proxy

environment:

ZOOKEEPER_HEAP_SIZE: "-Xmx256m -Xms256m"

KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"

ZOOKEEPER_CLIENT_PORT: 2181

ZOOKEEPER_SERVER_ID: 3

ZOOKEEPER_SERVERS: zk-1:2888:3888;zk-2:2888:3888;0.0.0.0:2888:3888

ports:

- 2183:2181

volumes:

- zk-3:/var/lib/zookeeper/data

deploy:

mode: replicated

replicas: 1

labels:

- com.df.notify=true

- com.df.serviceDomain=zk3.test.ru

- com.df.port=2181

kafka-1:

image: confluentinc/cp-kafka:4.0.0

hostname: kafka-1

networks:

- kafka_default

environment:

KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"

KAFKA_ZOOKEEPER_CONNECT: zk-1:2181,zk-2:2181,zk-3:2181

KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:9192

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT

KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

KAFKA_DEFAULT_REPLICATION_FACTOR: 3

KAFKA_JMX_PORT: 9181

KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9192,EXTERNAL://kafka.test.ru:9092

KAFKA_JMX_HOSTNAME: kafka-1

ports:

- 9092:9092

volumes:

- kafka-1:/var/lib/kafka/data

deploy:

mode: replicated

replicas: 1

kafka-2:

image: confluentinc/cp-kafka:4.0.0

hostname: kafka-2

networks:

- kafka_default

- proxy

environment:

KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"

KAFKA_ZOOKEEPER_CONNECT: zk-1:2181,zk-2:2181,zk-3:2181

KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:9192

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT

KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

KAFKA_DEFAULT_REPLICATION_FACTOR: 3

KAFKA_JMX_PORT: 9181

KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:9192,EXTERNAL://kafka.test.ru:9093

KAFKA_JMX_HOSTNAME: kafka-2

ports:

- 9093:9092

volumes:

- kafka-2:/var/lib/kafka/data

deploy:

mode: replicated

replicas: 1

kafka-3:

image: confluentinc/cp-kafka:4.0.0

hostname: kafka-3

networks:

- kafka_default

- proxy

environment:

KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"

KAFKA_ZOOKEEPER_CONNECT: zk-1:2181,zk-2:2181,zk-3:2181

KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:9192

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT

KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

KAFKA_DEFAULT_REPLICATION_FACTOR: 3

KAFKA_JMX_PORT: 9181

KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:9192,EXTERNAL://kafka.test.ru:9094

KAFKA_JMX_HOSTNAME: kafka-3

ports:

- 9094:9092

volumes:

- kafka-3:/var/lib/kafka/data

deploy:

mode: replicated

replicas: 1

kafka-manager:

image: sheepkiller/kafka-manager

hostname: kafka-manager

networks:

- kafka_default

- proxy

environment:

ZK_HOSTS: zk-1:2181,zk-2:2181,zk-3:2181

JMX_PORT: 9181

APPLICATION_SECRET: letmein

ports:

- '9000'

deploy:

mode: replicated

replicas: 1

labels:

- com.df.notify=true

- com.df.serviceDomain=kafkamgr.test.ru

- com.df.port=9000

networks:

kafka_default:

driver: overlay

proxy:

external: true

[/codesyntax]

деплоим наш стек:
docker stack deploy -c kafka.yml kafka

постоянные хранилища созданы в кластере ceph, также работает docker-flow-proxy, с помощью которого и прокидываем зоокипер.
менеджер kafka доступен по ссылке:
kafkamgr.test.ru

Zookeeper доступен по следующим доменным именам:
zk1.test.ru
zk2.test.ru
zk3.test.ru

Kafka доступна по следующим URL:
kafka.test.ru:9092
kafka.test.ru:9093
kafka.test.ru:9094

чтобы проверить работу,добавляем в /etc/hosts:
cat /etc/hosts
ip1 kafka.test.ru zk1.test.ru zk3.test.ru zk2.test.ru
ip2 kafka.test.ru zk1.test.ru zk3.test.ru zk2.test.ru
ip3 kafka.test.ru zk1.test.ru zk3.test.ru zk2.test.ru

и качаем:
wget https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.11-2.3.0.tgz
tar -xzvf kafka_2.11-2.3.0.tgz
cd kafka_2.11-2.3.0/bin/

создаём топик с именем TestTopic, количество реплик 3:
./kafka-topics.sh --create --zookeeper zk1.test.ru:2181,zk2.test.ru:2181,zk3.test.ru:2181 --replication-factor 3 --partitions 1 --topic TestTopic
Created topic TestTopic.

Посмотреть список топиков:
./kafka-topics.sh --list --zookeeper zk1.test.ru:2181
TestTopic
TutorialTopic
__consumer_offsets

создаём producer:
[root@node1 bin]# ./kafka-console-producer.sh --broker-list kafka.test.ru:9092 --topic TestTopic
>

подключаемся с помощью consumer к любому контейнеру с kafka:
[root@node3 bin]# ./kafka-console-consumer.sh --bootstrap-server kafka.test.ru:9093 --topic TestTopic --from-beginning

после этого можем передавать сообщения:
они появятся со стороны consumer: