Kubernetes. Deploy RabbitMQ cluster with Gitlab CI

Thank you for reading this post, don't forget to subscribe!

RabbitMQ — брокер сообщений, работающий по протоколу AMQP, позволяет создавать очереди из поступающих данных.

Основные переменные: название сервиса $YOUR_APP и $ KUBE_NAMESPASE – имя пространства в кластере kubernetes. 192.168.0.63 – это nexus докер репозиторий, для входа в него используются переменные, $NEXUS_USER и $NEXUS_PASS которые добавлены в variables в настройках проекта gitlab. Количество реплик – 3, минимально рекомендуемое для работы кластера. Rabbitmq является stateful приложением, в качестве хранилища монтируется файл с хостовой системы (hostPath), в условиях развертывания определенно ограничение, не больше одной реплики на один хост. Данная настройка может быть изменена. В скрипте также конкретно указан кластер k8s, если у вас k8s интегрирован в gitlab, то закомментируйте эти строки.

Создаем новый проект, и в нем создаем следующую структуру:

/cluster-mq
|--- manifests
      |--- mq-configmap.yaml
      |--- mq-pv.yaml
      |--- mq-pvc.yaml
      |--- mq-serviceaccount.yaml
      |--- mq-service-internal.yaml
      |--- mq-statefulset.yaml
|--- .gitlab-ci.yml

/cluster-mq

|--- manifests

|--- mq-configmap.yaml

|--- mq-pv.yaml

|--- mq-pvc.yaml

|--- mq-serviceaccount.yaml

|--- mq-service-internal.yaml

|--- mq-statefulset.yaml

|--- .gitlab-ci.yml

Содержимое файлов:

Скрипт развертывания в kubernetes
.gitlab-ci.yml

[codesyntax lang="php"]

image: alpine:latest
variables:
  YOUR_APP: mq
  KUBERNETES_VERSION: 1.11.7
  KUBE_NAMESPACE: your_namespace
stages:
  - production
production:
  stage: production
  image: docker:stable
  services:
    - docker:dind
  script:
    - install_dependencies
# подключаем конкретный кластер k8s
    - kubectl config set-cluster default-cluster --insecure-skip-tls-verify=true --server=${KUBE_URL}
    - kubectl config set-credentials default-admin --username=${KUBE_USER} --password=${KUBE_PASSWORD}
    - kubectl config set-context default-system --user=default-admin --namespace=${KUBE_NAMESPACE} --cluster=default-cluster
    - kubectl config use-context default-system
    - kubectl cluster-info
    - install_secrets
    - cd manifests/
    - sed -i "s/__YOUR_APP__/${YOUR_APP}/" mq-configmap.yaml \
mq-pv.yaml mq-pvc.yaml mq-serviceaccount.yaml mq-service-internal.yaml mq-statefulset.yaml
    - sed -i "s/__KUBE_NAMESPACE__/${KUBE_NAMESPACE}/" mq-configmap.yaml \
mq-pv.yaml mq-pvc.yaml mq-serviceaccount.yaml mq-service-internal.yaml mq-statefulset.yaml
    - kubectl apply -f mq-statefulset.yaml.yaml
    - kubectl patch -f \
statefulset.yaml -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ci-last-updated\":\"$(date +'%s')\"}}}}}"
    - kubectl apply -f mq-serviceaccount.yaml
    - kubectl apply -f mq-pv.yaml
    - kubectl apply -f mq-pvc.yaml
    - kubectl apply -f mq-service-internal.yaml
    - kubectl apply -f mq-configmap.yaml
    - kubectl rollout status -f mq-statefulset.yaml
    - add_mq_user
  environment:
    name: $YOUR_APP
    url: http://$YOUR_APP.$KUBE_INGRESS_BASE_DOMAIN
.auto_devops: &amp;auto_devops |
  function install_dependencies() {
    apk add -U curl
    curl -L -o /usr/bin/kubectl \
"https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
    chmod +x /usr/bin/kubectl
    kubectl version --client
  }
  function registry_login() {
    if [[ -n "$NEXUS_USER" ]]; then
      echo "Logging to Nexus Container Registry"
      docker login -u "$NEXUS_USER" -p "$NEXUS_PASS" 192.168.0.63
      echo ""
    fi
  }
  function install_secrets() {
    kubectl create secret -n $KUBE_NAMESPACE \
      docker-registry regsecret \
      --docker-server="192.168.0.63" \
      --docker-username="$NEXUS_USER" \
      --docker-password="$NEXUS_PASS" \
      --docker-email=" gitlab@nafanasev.local" \
      -o yaml --dry-run | kubectl replace -n $KUBE_NAMESPACE --force -f -
  }
# укажите логин и пароль для rabbitmq кластера
  function add_mq_user() {
    kubectl -n $KUBE_NAMESPACE exec "$YOUR_APP"-0 \
    -- bash -c "rabbitmqctl add_user ЛОГИН_ДЛЯ_MQ ПАРОЛЬ_MQ; rabbitmqctl set_user_tags ЛОГИН_ДЛЯ_MQ administrator; \
    rabbitmqctl set_permissions -p / ЛОГИН_ДЛЯ_MQ '.*' '.*' '.*'; echo \"rabbitmqctl list_user_permissions ЛОГИН_ДЛЯ_MQ\""
  }
 
before_script:
  - *auto_devops

image: alpine:latest

variables:

YOUR_APP: mq

KUBERNETES_VERSION: 1.11.7

KUBE_NAMESPACE: your_namespace

stages:

- production

production:

stage: production

image: docker:stable

services:

- docker:dind

script:

- install_dependencies

# подключаем конкретный кластер k8s

- kubectl config set-cluster default-cluster --insecure-skip-tls-verify=true --server=${KUBE_URL}

- kubectl config set-credentials default-admin --username=${KUBE_USER} --password=${KUBE_PASSWORD}

- kubectl config set-context default-system --user=default-admin --namespace=${KUBE_NAMESPACE} --cluster=default-cluster

- kubectl config use-context default-system

- kubectl cluster-info

- install_secrets

- cd manifests/

- sed -i "s/__YOUR_APP__/${YOUR_APP}/" mq-configmap.yaml \

mq-pv.yaml mq-pvc.yaml mq-serviceaccount.yaml mq-service-internal.yaml mq-statefulset.yaml

- sed -i "s/__KUBE_NAMESPACE__/${KUBE_NAMESPACE}/" mq-configmap.yaml \

mq-pv.yaml mq-pvc.yaml mq-serviceaccount.yaml mq-service-internal.yaml mq-statefulset.yaml

- kubectl apply -f mq-statefulset.yaml.yaml

- kubectl patch -f \

statefulset.yaml -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ci-last-updated\":\"$(date +'%s')\"}}}}}"

- kubectl apply -f mq-serviceaccount.yaml

- kubectl apply -f mq-pv.yaml

- kubectl apply -f mq-pvc.yaml

- kubectl apply -f mq-service-internal.yaml

- kubectl apply -f mq-configmap.yaml

- kubectl rollout status -f mq-statefulset.yaml

- add_mq_user

environment:

url: http://$YOUR_APP.$KUBE_INGRESS_BASE_DOMAIN

.auto_devops: &auto_devops |

function install_dependencies() {

apk add -U curl

curl -L -o /usr/bin/kubectl \

"https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"

chmod +x /usr/bin/kubectl

kubectl version --client

}

function registry_login() {

if [[ -n "$NEXUS_USER" ]]; then

echo "Logging to Nexus Container Registry"

docker login -u "$NEXUS_USER" -p "$NEXUS_PASS" 192.168.0.63

echo ""

}

function install_secrets() {

kubectl create secret -n $KUBE_NAMESPACE \

docker-registry regsecret \

--docker-server="192.168.0.63" \

--docker-username="$NEXUS_USER" \

--docker-password="$NEXUS_PASS" \

--docker-email=" gitlab@nafanasev.local" \

-o yaml --dry-run | kubectl replace -n $KUBE_NAMESPACE --force -f -

}

# укажите логин и пароль для rabbitmq кластера

function add_mq_user() {

kubectl -n $KUBE_NAMESPACE exec "$YOUR_APP"-0 \

-- bash -c "rabbitmqctl add_user ЛОГИН_ДЛЯ_MQ ПАРОЛЬ_MQ; rabbitmqctl set_user_tags ЛОГИН_ДЛЯ_MQ administrator; \

rabbitmqctl set_permissions -p / ЛОГИН_ДЛЯ_MQ '.*' '.*' '.*'; echo \"rabbitmqctl list_user_permissions ЛОГИН_ДЛЯ_MQ\""

}

before_script:

- *auto_devops

[/codesyntax]

Создаём ServiceAccount для нашего кластера и выдаём ему права на чтение Endpoints K8s.
mq-serviceaccount.yaml

[codesyntax lang="php"]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: mq
  namespace: __KUBE_NAMESPACE__
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: endpoint-reader
  namespace: __KUBE_NAMESPACE__
rules:
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: endpoint-reader
  namespace: __KUBE_NAMESPACE__
subjects:
- kind: ServiceAccount
  name: mq
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: endpoint-reader

---

apiVersion: v1

kind: ServiceAccount

metadata:

namespace: __KUBE_NAMESPACE__

---

kind: Role

apiVersion: rbac.authorization.k8s.io/v1beta1

metadata:

namespace: __KUBE_NAMESPACE__

rules:

- apiGroups: [""]

resources: ["endpoints"]

verbs: ["get"]

---

kind: RoleBinding

apiVersion: rbac.authorization.k8s.io/v1beta1

metadata:

namespace: __KUBE_NAMESPACE__

subjects:

- kind: ServiceAccount

roleRef:

apiGroup: rbac.authorization.k8s.io

kind: Role

[/codesyntax]

Создаем персистентное хранилище - hostPath
mq-pv.yaml

[codesyntax lang="php"]

kind: PersistentVolume
apiVersion: v1
metadata:
  name: mq-pv
  namespace: __KUBE_NAMESPACE__
  labels:
    type: local
spec:
  storageClassName: mq-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/opt/mq-pv"

kind: PersistentVolume

apiVersion: v1

metadata:

namespace: __KUBE_NAMESPACE__

labels:

type: local

spec:

storageClassName: mq-pv

capacity:

storage: 10Gi

accessModes:

- ReadWriteOnce

persistentVolumeReclaimPolicy: Recycle

hostPath:

path: "/opt/mq-pv"

[/codesyntax]

Привязываем volume claim к хранилищу
mq-pvc.yaml

[codesyntax lang="php"]

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: mq-pvc
  namespace: __KUBE_NAMESPACE__
spec:
  storageClassName: mq-pv
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi

kind: PersistentVolumeClaim

apiVersion: v1

metadata:

namespace: __KUBE_NAMESPACE__

spec:

storageClassName: mq-pv

accessModes:

- ReadWriteOnce

resources:

requests:

storage: 10Gi

[/codesyntax]

Создаем конфигурационный файл rabbitmq. Укажите порт для kubernetes (443 или 6443) и замените "your_namespace"
mq-configmap.yaml

[codesyntax lang="php"]

apiVersion: v1
kind: ConfigMap
metadata:
  name: mq-config
  namespace: __KUBE_NAMESPACE__
data:
  enabled_plugins: |
      [rabbitmq_management,rabbitmq_peer_discovery_k8s].
  rabbitmq.conf: |
      cluster_formation.peer_discovery_backend  = rabbit_peer_discovery_k8s
      cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
      cluster_formation.k8s.port = 6443
      cluster_formation.k8s.address_type = hostname
      cluster_formation.node_cleanup.interval = 10
      cluster_formation.node_cleanup.only_log_warning = true
      cluster_partition_handling = autoheal
      queue_master_locator=min-masters
      cluster_formation.randomized_startup_delay_range.min = 0
      cluster_formation.randomized_startup_delay_range.max = 2
      cluster_formation.k8s.service_name = mq-internal
      cluster_formation.k8s.hostname_suffix = mq-internal.your_namespace.svc.cluster.local

apiVersion: v1

kind: ConfigMap

metadata:

namespace: __KUBE_NAMESPACE__

data:

enabled_plugins: |

[rabbitmq_management,rabbitmq_peer_discovery_k8s].

rabbitmq.conf: |

cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s

cluster_formation.k8s.host = kubernetes.default.svc.cluster.local

cluster_formation.k8s.port = 6443

cluster_formation.k8s.address_type = hostname

cluster_formation.node_cleanup.interval = 10

cluster_formation.node_cleanup.only_log_warning = true

cluster_partition_handling = autoheal

queue_master_locator=min-masters

cluster_formation.randomized_startup_delay_range.min = 0

cluster_formation.randomized_startup_delay_range.max = 2

cluster_formation.k8s.service_name = mq-internal

cluster_formation.k8s.hostname_suffix = mq-internal.your_namespace.svc.cluster.local

[/codesyntax]

Создаем входную сетевую точку
mq-service-internal.yaml

[codesyntax lang="php"]

kind: Service
apiVersion: v1
metadata:
  name: mq-internal
  namespace: __KUBE_NAMESPACE__
  labels:
    app: mq
spec:
  ports:
   - name: http
     protocol: TCP
     port: 15672
   - name: amqp
     protocol: TCP
     port: 5672
  selector:
    app: mq
  type: ClusterIP

kind: Service

apiVersion: v1

metadata:

namespace: __KUBE_NAMESPACE__

labels:

app: mq

spec:

ports:

- name: http

protocol: TCP

port: 15672

- name: amqp

protocol: TCP

port: 5672

selector:

app: mq

type: ClusterIP

[/codesyntax]

Развертываем stateful приложение
RABBITMQ_ERLANG_COOKIE – это «пароль» для связи между нодами кластера (установите свой)
mq-statefulset.yaml

[codesyntax lang="php"]

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: mq
  namespace: __KUBE_NAMESPACE__
spec:
  serviceName: mq-internal
  replicas: 3
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: mq
      annotations:
        scheduler.alpha.kubernetes.io/affinity: &gt;
            {
              "podAntiAffinity": {
                "requiredDuringSchedulingIgnoredDuringExecution": [{
                  "labelSelector": {
                    "matchExpressions": [{
                      "key": "app",
                      "operator": "In",
                      "values": ["mq"]
                    }]
                  },
                  "topologyKey": "kubernetes.io/hostname"
                }]
              }
            }
    spec:
      serviceAccountName: mq
      terminationGracePeriodSeconds: 10
      containers:       
      - name: mq
        image: rabbitmq:3.7
        volumeMounts:
          - name: mq-config
            mountPath: /etc/rabbitmq
          - name: mq-pvc
            mountPath: /var/lib/rabbitmq/mnesia
        ports:
          - name: http
            protocol: TCP
            containerPort: 15672
          - name: amqp
            protocol: TCP
            containerPort: 5672
        resources:
          requests:
            memory: "128Mi"
            cpu: "0.1"
          limits:
            memory: "1024Mi"
            cpu: "0.5" 
        livenessProbe:
          exec:
            command: ["rabbitmq-diagnostics", "node_health_check"]
          initialDelaySeconds: 70
          periodSeconds: 70
          timeoutSeconds: 10
        readinessProbe:
          exec:
            command: ["rabbitmqctl", "status"]
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 10
        imagePullPolicy: Always
        env:
          - name: MY_POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
          - name: HOSTNAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
          - name: RABBITMQ_USE_LONGNAME
            value: "true"
          - name: RABBITMQ_NODENAME
            value: "rabbit@$(HOSTNAME).mq-internal.$(NAMESPACE).svc.cluster.local"
          - name: K8S_SERVICE_NAME
            value: "mq-internal"
          - name: RABBITMQ_ERLANG_COOKIE
            value: "mq"
      volumes:
        - name: mq-config
          configMap:
            name: mq-config
            items:
            - key: rabbitmq.conf
              path: rabbitmq.conf
            - key: enabled_plugins
              path: enabled_plugins
        - name: mq-pvc
          persistentVolumeClaim:
            claimName: mq-pvc

100

101

apiVersion: apps/v1beta1

kind: StatefulSet

metadata:

namespace: __KUBE_NAMESPACE__

spec:

serviceName: mq-internal

replicas: 3

updateStrategy:

type: RollingUpdate

template:

metadata:

labels:

app: mq

annotations:

scheduler.alpha.kubernetes.io/affinity: >

{

"podAntiAffinity": {

"requiredDuringSchedulingIgnoredDuringExecution": [{

"labelSelector": {

"matchExpressions": [{

"key": "app",

"operator": "In",

"values": ["mq"]

}]

"topologyKey": "kubernetes.io/hostname"

}]

}

spec:

serviceAccountName: mq

terminationGracePeriodSeconds: 10

containers:

- name: mq

image: rabbitmq:3.7

volumeMounts:

- name: mq-config

mountPath: /etc/rabbitmq

- name: mq-pvc

mountPath: /var/lib/rabbitmq/mnesia

ports:

- name: http

protocol: TCP

containerPort: 15672

- name: amqp

protocol: TCP

containerPort: 5672

resources:

requests:

memory: "128Mi"

cpu: "0.1"

limits:

memory: "1024Mi"

cpu: "0.5"

livenessProbe:

exec:

command: ["rabbitmq-diagnostics", "node_health_check"]

initialDelaySeconds: 70

periodSeconds: 70

timeoutSeconds: 10

readinessProbe:

exec:

command: ["rabbitmqctl", "status"]

initialDelaySeconds: 10

periodSeconds: 10

timeoutSeconds: 10

imagePullPolicy: Always

env:

- name: MY_POD_IP

valueFrom:

fieldRef:

fieldPath: status.podIP

- name: HOSTNAME

valueFrom:

fieldRef:

fieldPath: metadata.name

- name: NAMESPACE

valueFrom:

fieldRef:

fieldPath: metadata.namespace

- name: RABBITMQ_USE_LONGNAME

value: "true"

- name: RABBITMQ_NODENAME

value: "rabbit@$(HOSTNAME).mq-internal.$(NAMESPACE).svc.cluster.local"

- name: K8S_SERVICE_NAME

value: "mq-internal"

- name: RABBITMQ_ERLANG_COOKIE

value: "mq"

volumes:

- name: mq-config

configMap:

items:

- key: rabbitmq.conf

path: rabbitmq.conf

- key: enabled_plugins

path: enabled_plugins

- name: mq-pvc

persistentVolumeClaim:

claimName: mq-pvc

[/codesyntax]

Пн	Вт	Ср	Чт	Пт	Сб	Вс
1	2	3	4	5	6	7
8	9	10	11	12	13	14
15	16	17	18	19	20	21
22	23	24	25	26	27	28
29	30

Centos

Kubernetes. Deploy RabbitMQ cluster with Gitlab CI

https://github.com/midnight47/