526互联

strimzi operator 部署kafka集群

发布时间 2023-11-21 11:04:43作者: 小吉猫

环境说明

本环境使用了单节点、临时存储集群的kafka-ephemeral-single配置。线上环境推荐kafka-persistent.yaml配置并修改storage配置为自动创建pv/pvc类型。

配置清单说明

1. kafka-ephemeral-single.yaml：非持久化存储，单节点集群；
2. kafka-ephemeral.yaml：非持久化存储，多节点集群；
3. kafka-jbod.yaml：jbod存储，多节点集群；
4. kafka-persistent-single.yaml：持久化存储，单节点集群；
5. kafka-persistent.yaml ：持久化存储，多节点集群；

创建名称空间

# kubectl create ns kafka

创建 strimzi-cluster-operator

创建 strimzi-cluster-operator 资源

# kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka
rolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator created
configmap/strimzi-cluster-operator created
clusterrole.rbac.authorization.k8s.io/strimzi-kafka-broker created
customresourcedefinition.apiextensions.k8s.io/kafkaconnects.kafka.strimzi.io created
clusterrole.rbac.authorization.k8s.io/strimzi-entity-operator created
customresourcedefinition.apiextensions.k8s.io/kafkatopics.kafka.strimzi.io created
clusterrole.rbac.authorization.k8s.io/strimzi-cluster-operator-leader-election created
customresourcedefinition.apiextensions.k8s.io/kafkamirrormaker2s.kafka.strimzi.io created
rolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator-entity-operator-delegation created
clusterrole.rbac.authorization.k8s.io/strimzi-cluster-operator-global created
customresourcedefinition.apiextensions.k8s.io/kafkabridges.kafka.strimzi.io created
serviceaccount/strimzi-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator created
clusterrole.rbac.authorization.k8s.io/strimzi-kafka-client created
customresourcedefinition.apiextensions.k8s.io/kafkausers.kafka.strimzi.io created
clusterrolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator-kafka-broker-delegation created
customresourcedefinition.apiextensions.k8s.io/kafkaconnectors.kafka.strimzi.io created
clusterrole.rbac.authorization.k8s.io/strimzi-cluster-operator-watched created
customresourcedefinition.apiextensions.k8s.io/kafkas.kafka.strimzi.io created
rolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator-watched created
customresourcedefinition.apiextensions.k8s.io/kafkarebalances.kafka.strimzi.io created
rolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator-leader-election created
clusterrole.rbac.authorization.k8s.io/strimzi-cluster-operator-namespaced created
customresourcedefinition.apiextensions.k8s.io/kafkamirrormakers.kafka.strimzi.io created
customresourcedefinition.apiextensions.k8s.io/kafkanodepools.kafka.strimzi.io created
customresourcedefinition.apiextensions.k8s.io/strimzipodsets.core.strimzi.io created
clusterrolebinding.rbac.authorization.k8s.io/strimzi-cluster-operator-kafka-client-delegation created
deployment.apps/strimzi-cluster-operator created

查看 strimzi-cluster-operator API资源

# kubectl api-resources |grep kafka
kafkabridges                      kb                                              kafka.strimzi.io/v1beta2                    true         KafkaBridge
kafkaconnectors                   kctr                                            kafka.strimzi.io/v1beta2                    true         KafkaConnector
kafkaconnects                     kc                                              kafka.strimzi.io/v1beta2                    true         KafkaConnect
kafkamirrormaker2s                kmm2                                            kafka.strimzi.io/v1beta2                    true         KafkaMirrorMaker2
kafkamirrormakers                 kmm                                             kafka.strimzi.io/v1beta2                    true         KafkaMirrorMaker
kafkanodepools                    knp                                             kafka.strimzi.io/v1beta2                    true         KafkaNodePool
kafkarebalances                   kr                                              kafka.strimzi.io/v1beta2                    true         KafkaRebalance
kafkas                            k                                               kafka.strimzi.io/v1beta2                    true         Kafka
kafkatopics                       kt                                              kafka.strimzi.io/v1beta2                    true         KafkaTopic
kafkausers                        ku                                              kafka.strimzi.io/v1beta2                    true         KafkaUser

查看 strimzi-cluster-operator pod

# kubectl get pods -n kafka
NAME                                        READY   STATUS    RESTARTS   AGE
strimzi-cluster-operator-5d6f48c6f9-s56hr   1/1     Running   0          5m1s

创建 kafka 集群

kafka-ephemeral-single.yaml

apiVersion: kafka.strimzi.io/v1beta2
kind: Kafka
metadata:
  name: my-cluster
  namespace: kafka
spec:
  kafka:
    version: 3.6.0
    replicas: 1
    listeners:
      - name: plain
        port: 9092
        type: internal
        tls: false
      - name: tls
        port: 9093
        type: internal
        tls: true
    config:
      offsets.topic.replication.factor: 1
      transaction.state.log.replication.factor: 1
      transaction.state.log.min.isr: 1
      default.replication.factor: 1
      min.insync.replicas: 1
      inter.broker.protocol.version: "3.6"
    storage:
      type: ephemeral
  zookeeper:
    replicas: 3
    storage:
      type: ephemeral
  entityOperator:
    topicOperator: {}
    userOperator: {}

创建 kafka 集群

# kubectl apply -f kafka-ephemeral-single.yaml
kafka.kafka.strimzi.io/my-cluster created

查看 kafka pod

# kubectl get pods -n kafka
NAME                                          READY   STATUS    RESTARTS        AGE
my-cluster-entity-operator-75765b6dbb-l5s9q   3/3     Running   0               56s
my-cluster-kafka-0                            1/1     Running   0               81s
my-cluster-zookeeper-0                        1/1     Running   0               2m28s
my-cluster-zookeeper-1                        1/1     Running   0               2m28s
my-cluster-zookeeper-2                        1/1     Running   0               2m28s
strimzi-cluster-operator-5d6f48c6f9-vrfjp     1/1     Running   0               3m11s

测试 kafka 集群

发送消息

# kubectl -n kafka run kafka-producer -ti --image=quay.io/strimzi/kafka:0.38.0-kafka-3.6.0 --rm=true --restart=Never -- bin/kafka-console-producer.sh --bootstrap-server my-cluster-kafka-bootstrap:9092 --topic my-topic
If you don't see a command prompt, try pressing enter.
>hello
[2023-11-21 02:43:38,971] WARN [Producer clientId=console-producer] Error while fetching metadata with correlation id 4 : {my-topic=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
>kafka
>

接收消息

# kubectl -n kafka run kafka-consumer -ti --image=quay.io/strimzi/kafka:0.38.0-kafka-3.6.0 --rm=true --restart=Never -- bin/kafka-console-consumer.sh --bootstrap-server my-cluster-kafka-bootstrap:9092 --topic my-topic --from-beginning
If you don't see a command prompt, try pressing enter.
hello
kafka

删除 kafka 集群

# kubectl delete -f kafka-ephemeral-single.yaml
kafka.kafka.strimzi.io "my-cluster" deleted

删除 strimzi-cluster-operator

kubectl -n kafka delete -f 'https://strimzi.io/install/latest?namespace=kafka'
rolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator" deleted
configmap "strimzi-cluster-operator" deleted
Warning: deleting cluster-scoped resources, not scoped to the provided namespace
clusterrole.rbac.authorization.k8s.io "strimzi-kafka-broker" deleted
customresourcedefinition.apiextensions.k8s.io "kafkaconnects.kafka.strimzi.io" deleted
clusterrole.rbac.authorization.k8s.io "strimzi-entity-operator" deleted
customresourcedefinition.apiextensions.k8s.io "kafkatopics.kafka.strimzi.io" deleted
clusterrole.rbac.authorization.k8s.io "strimzi-cluster-operator-leader-election" deleted
customresourcedefinition.apiextensions.k8s.io "kafkamirrormaker2s.kafka.strimzi.io" deleted
rolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator-entity-operator-delegation" deleted
clusterrole.rbac.authorization.k8s.io "strimzi-cluster-operator-global" deleted
customresourcedefinition.apiextensions.k8s.io "kafkabridges.kafka.strimzi.io" deleted
serviceaccount "strimzi-cluster-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator" deleted
clusterrole.rbac.authorization.k8s.io "strimzi-kafka-client" deleted
customresourcedefinition.apiextensions.k8s.io "kafkausers.kafka.strimzi.io" deleted
clusterrolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator-kafka-broker-delegation" deleted
customresourcedefinition.apiextensions.k8s.io "kafkaconnectors.kafka.strimzi.io" deleted
clusterrole.rbac.authorization.k8s.io "strimzi-cluster-operator-watched" deleted
customresourcedefinition.apiextensions.k8s.io "kafkas.kafka.strimzi.io" deleted
rolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator-watched" deleted
customresourcedefinition.apiextensions.k8s.io "kafkarebalances.kafka.strimzi.io" deleted
rolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator-leader-election" deleted
clusterrole.rbac.authorization.k8s.io "strimzi-cluster-operator-namespaced" deleted
customresourcedefinition.apiextensions.k8s.io "kafkamirrormakers.kafka.strimzi.io" deleted
customresourcedefinition.apiextensions.k8s.io "kafkanodepools.kafka.strimzi.io" deleted
customresourcedefinition.apiextensions.k8s.io "strimzipodsets.core.strimzi.io" deleted
clusterrolebinding.rbac.authorization.k8s.io "strimzi-cluster-operator-kafka-client-delegation" deleted
deployment.apps "strimzi-cluster-operator" deleted

参考文档

https://strimzi.io/quickstarts/

https://github.com/strimzi/strimzi-kafka-operator/tree/0.38.0/examples