author: caijunhui
date: 2019-08-29

本文介绍如何在k8s环境下部署rabbitmq

镜像使用rabbitmq:3.7-management-alpine，从3.7.0开始rabbitmq使用了peer discovery， 不使用autoscale。

1. 创建rabbitmq的namespce

   apiVersion: v1
   kind: Namespace
   metadata:
     name: rabbitmq
   

2. rdbc

   ---
   apiVersion: v1
   kind: ServiceAccount
   metadata:
     name: rabbitmq 
     namespace: rabbitmq 
   ---
   apiVersion: rbac.authorization.k8s.io/v1beta1
   kind: Role
   metadata:
     name: endpoint-reader
     namespace: rabbitmq 
   rules:
   - apiGroups: [""]
     resources: ["endpoints"]
     verbs: ["get"]
   ---
   kind: RoleBinding
   apiVersion: rbac.authorization.k8s.io/v1beta1
   metadata:
     name: endpoint-reader
     namespace: rabbitmq
   subjects:
   - kind: ServiceAccount
     name: rabbitmq
   roleRef:
     apiGroup: rbac.authorization.k8s.io
     kind: Role
     name: endpoint-reader
   

3. rabbitmq数据持久化

   apiVersion: v1
   kind: PersistentVolumeClaim
   metadata:
     name: rabbitmq-pvc
     namespace: rabbitmq
   spec:
     accessModes:
       - ReadWriteMany
     resources:  
       requests:
         storage: 5Gi
     storageClassName: managed-nfs-storage
   

4. Deploy

   # NodePort Servce，以便通过集群节点访问rabbitmq
   kind: Service
   apiVersion: v1
   metadata:
     namespace: rabbitmq
     name: rabbitmq-svc
     labels:
       app: rabbitmq
       type: LoadBalancer  
   spec:
     type: NodePort
     ports:
      - name: rabbitmq-mgmt-port
        protocol: TCP
        port: 15672
        targetPort: 15672
        nodePort: 31672
      - name: rabbitmq-amqp-port
        protocol: TCP
        port: 5672
        targetPort: 5672
        nodePort: 30672
     selector:
       app: rabbitmq
       
   ---
   apiVersion: v1
   kind: Service
   metadata:
     name: rabbitmq
     namespace: rabbitmq
     labels:
       app: rabbitmq
   spec:
     clusterIP: None
     #用Headless Service去做Pod的hostname访问，需要等Pod和Service都启动后才能访问，而readiness探针还没等DNS正常就去探查服务是否可用，所以才会误认为服务不可达，最终无法启动Pod。解决办法是给Headless Service设置publishNotReadyAddresses: true
     publishNotReadyAddresses: true
     ports:
     - port: 5672
       protocol: TCP
       name: amqp
     - port: 4369
       name: epmd
     - port: 15672
       protocol: TCP
       name: http
     selector:
       app: rabbitmq
   ---
   apiVersion: v1
   kind: ConfigMap
   metadata:
     name: rabbitmq-cm
     namespace: rabbitmq
   data:
     enabled_plugins: |
         [rabbitmq_management,rabbitmq_peer_discovery_k8s].
     rabbitmq.conf: |
         ## Cluster formation. See https://www.rabbitmq.com/cluster-formation.html to learn more.
         cluster_formation.peer_discovery_backend  = rabbit_peer_discovery_k8s
         cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
         # 必须设置service_name，否则Pod无法正常启动，这里设置后可以不设置statefulset下env中的K8S_SERVICE_NAME变量
         cluster_formation.k8s.service_name = rabbitmq
         # 必须设置hostname_suffix，否则节点不能成为集群
         cluster_formation.k8s.hostname_suffix = .rabbitmq.rabbitmq.svc.cluster.local
         ## Should RabbitMQ node name be computed from the pod's hostname or IP address?
         ## IP addresses are not stable, so using [stable] hostnames is recommended when possible.
         ## Set to "hostname" to use pod hostnames.
         ## When this value is changed, so should the variable used to set the RABBITMQ_NODENAME
         ## environment variable.
         ## hostname or ip
         cluster_formation.k8s.address_type = hostname
         ## How often should node cleanup checks run?
         cluster_formation.node_cleanup.interval = 30
         cluster_formation.randomized_startup_delay_range.min = 0
         cluster_formation.randomized_startup_delay_range.max = 2
         ## Set to false if automatic removal of unknown/absent nodes
         ## is desired. This can be dangerous, see
         ##  * https://www.rabbitmq.com/cluster-formation.html#node-health-checks-and-cleanup
         ##  * https://groups.google.com/forum/#!msg/rabbitmq-users/wuOfzEywHXo/k8z_HWIkBgAJ
         cluster_formation.node_cleanup.only_log_warning = true
         cluster_partition_handling = autoheal
         ## See https://www.rabbitmq.com/ha.html#master-migration-data-locality
         queue_master_locator=min-masters
         ## See https://www.rabbitmq.com/access-control.html#loopback-users
         loopback_users.guest = false
      
   ---
   apiVersion: apps/v1beta1
   kind: StatefulSet
   metadata:
     name: rabbitmq
     namespace: rabbitmq
   spec:
     serviceName: rabbitmq
     selector:
       matchLabels:
         app: rabbitmq
     replicas: 3
     template:
       metadata:
         labels:
           app: rabbitmq
         annotations:
                 scheduler.alpha.kubernetes.io/affinity: >
                     {
                       "podAntiAffinity": {
                         "requiredDuringSchedulingIgnoredDuringExecution": [{
                           "labelSelector": {
                             "matchExpressions": [{
                               "key": "app",
                               "operator": "In",
                               "values": ["rabbitmq"]
                             }]
                           },
                           "topologyKey": "kubernetes.io/hostname"
                         }]
                       }
                     }
       spec:
         serviceAccountName: rabbitmq
         terminationGracePeriodSeconds: 10
         containers:        
         - name: rabbitmq-k8s
           image: rabbitmq:3.7-management-alpine
           imagePullPolicy: IfNotPresent
           volumeMounts:
             - name: config-volume
               mountPath: /etc/rabbitmq
             - name: rabbitmq-data
             	mountPath: /var/lib/rabbitmq/data
           ports:
             - name: http
               protocol: TCP
               containerPort: 15672
             - name: amqp
               protocol: TCP
               containerPort: 5672
             - containerPort: 4369
             - containerPort: 25672
           livenessProbe:
             exec:
               command: ["rabbitmqctl", "status"]
             initialDelaySeconds: 60
             # See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations.
             periodSeconds: 60
             timeoutSeconds: 15
           readinessProbe:
             exec:
               command: ["rabbitmqctl", "status"]
             initialDelaySeconds: 20
             periodSeconds: 60
             timeoutSeconds: 10
           env:
             - name: MY_POD_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
             - name: MY_POD_IP
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
             - name: RABBITMQ_USE_LONGNAME
               value: "true"
             # - name: K8S_SERVICE_NAME
             #  value: "rabbitmq"
             # See a note on cluster_formation.k8s.address_type in the config file section
             - name: RABBITMQ_NODENAME
               value: "rabbit@$(MY_POD_NAME).rabbitmq.rabbitmq.svc.cluster.local"
             - name: RABBITMQ_ERLANG_COOKIE
               value: "Zk93VStwK0g3ZE5KNkxlT0lRQ0V2S3BTNXBockk0UU9QeVNrSXdRSGM0RT0K"
             #  valueFrom:
             #    secretKeyRef:
             #      name: erlang.cookie
             #      key: erlang.cookie 
         volumes:
           - name: config-volume
             configMap:
               name: rabbitmq-cm
               items:
               - key: rabbitmq.conf
                 path: rabbitmq.conf
               - key: enabled_plugins
                 path: enabled_plugins
           - name: rabbitmq-data
           	persistentVolumeClaim:
               claimName: rabbitmq-pvc