目标

安装3个副本的mysql集群,数据存储在ceph集群中

环境

  • kubernetes集群
master  10.115.223.121
node1   10.115.223.122
node2   10.115.223.123
  • ceph集群monitor
kvm-ceph1   10.115.223.201
kvm-ceph2   10.115.223.202
kvm-ceph3   10.115.223.203
ceph-deploy 10.115.223.200

准备工作

所有k8s节点运行以下命令

yum install -y ceph-common

创建名称为test的Namespace

# 创建test_namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
   name: test
   labels:
        name: test


kubectl create -f test_namespace.yaml

配置ceph集群客户端访问secret

从ceph_deploy获取ceph secret

grep key /etc/ceph/ceph.client.admin.keyring |awk '{printf "%s", $NF}'|base64

QVFBTnpYdGJvUWJ2Q2hBQUNqY3JHczJaeDQrR1lqT0kxNTJmU3c9PQ==

“QVFBTnpYdGJvUWJ2Q2hBQUNqY3JHczJaeDQrR1lqT0kxNTJmU3c9PQ==” 就是ceph集群client的秘钥

创建test_ceph_client_secret.yaml

apiVersion: v1
kind: Secret
metadata:
    name: ceph-secret
    namespace: test
type: "kubernetes.io/rbd"  
data:
    key: QVFBTnpYdGJvUWJ2Q2hBQUNqY3JHczJaeDQrR1lqT0kxNTJmU3c9PQ==

kubectl create -f test_ceph_client_secret.yaml

在ceph集群中配置pool

# 在kvm-ceph1 创建pool  k8s_test
ceph osd pool create k8s_test 8 8 replicated

创建 StorageClass

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: ceph-storage
provisioner: ceph.com/rbd
parameters:
  monitors: 10.115.223.201,10.115.223.202,10.115.223.203
  pool: k8s_test
  adminId: admin
  adminSecretNamespace: test
  adminSecretName: ceph-secret
  userId: admin
  userSecretName: ceph-secret
  imageFormat: "2"
  imageFeatures: layering

创建 rbd-provisioner

#rabc 授权配置如下
#方法一
kubectl create serviceaccount rbd-provisioner -n test
kubectl create clusterrolebinding ceph-cluster-rule --clusterrole=cluster-admin --serviceaccount=test:rbd-provisioner
#方法二
kind: ServiceAccount
apiVersion: v1
metadata:
  name: rbd-provisioner
  namespace: test
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rbd-provisioner
subjects:
- kind: ServiceAccount
  name: rbd-provisioner
  namespace: test
roleRef:
  kind: ClusterRole
  name: system:controller:persistent-volume-binder
  apiGroup: rbac.authorization.k8s.io
---

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: rbd-provisioner
  namespace: test
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: rbd-provisioner
    spec:
      containers:
      - name: rbd-provisioner
        image: "quay.io/external_storage/rbd-provisioner:v0.1.0"
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/rbd
      serviceAccountName: rbd-provisioner

创建 MySQL secret

创建 mysql-secret.yaml 内容如下:

#mysql_root_password
echo -n 'Qwer1234'  |base64
#mysql_user
echo -n 'admin'  |base64
#mysql_user_password
echo -n 'Admin1234'  |base64

apiVersion: v1
kind: Secret
metadata:
  name: mysql-secrets
  namespace: test
  labels:
    app: mysql
data:
  mysql_root_password: UXdlcjEyMzQ=
  mysql_user: YWRtaW4=
  mysql_user_password: QWRtaW4xMjM0

创建MySQL service

创建mysql-service.yaml 内容如下:

apiVersion: v1
kind: Service
metadata:
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
  name: mysql
  namespace: test
  labels:
    app: mysql
    tier: data
spec:
  ports:
  - port: 3306
    name: mysql
  clusterIP: None
  selector:
    app: mysql

创建MySQL StatefulSet

创建mysql-statefulSet.yaml

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: mysql
  namespace: test
spec:
  serviceName: "mysql"
  replicas: 3
  template:
    metadata:
      labels:
        app: mysql
      annotations:
        pod.alpha.kubernetes.io/initialized: "true"
    spec:
      securityContext:
        runAsUser: 27
        fsGroup: 27
      containers:
      - name: mysql
        image: adfinissygroup/k8s-mariadb-galera-centos:v004
        imagePullPolicy: Always
        securityContext:
          runAsNonRoot: true
        ports:
        - containerPort: 3306
          name: mysql
        - containerPort: 4444
          name: sst
        - containerPort: 4567
          name: replication
        - containerPort: 4568
          name: ist
        readinessProbe:
          exec:
            command:
            - /usr/share/container-scripts/mysql/readiness-probe.sh
          initialDelaySeconds: 15
          timeoutSeconds: 5
        volumeMounts:
        - name: datadir
          mountPath: /var/lib/mysql
          subPath: data
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: MYSQL_USER
          valueFrom:
            secretKeyRef:
              name: mysql-secrets
              key: mysql_user
        - name: MYSQL_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mysql-secrets
              key: mysql_user_password
        - name: MYSQL_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mysql-secrets
              key: mysql_root_password
  volumeClaimTemplates:
  - metadata:
      name: datadir
      annotations:
        volume.beta.kubernetes.io/storage-class: "ceph-storage"
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1Gi

验证结果

# 查看pod
[root@node121 ~]# kubectl get pod -n test
NAME                               READY     STATUS    RESTARTS   AGE
mysql-0                            1/1       Running   0          1h
mysql-1                            1/1       Running   0          1h
mysql-2                            1/1       Running   0          1h
rbd-provisioner-5b4ff69c9f-gt7w4   1/1       Running   0          22h

# 使用mysql客户端访问
kubectl run -it --rm --image=mysql:5.6 --restart=Never mysql-client -- mysql -h mysql -p{yourpasswd}
#已存在客户端时,使用一下命令直接进入mysql客户端
kubectl exec -it  mysql-client -n test -- mysql -h mysql -p{yourpasswd}

# 登录mysql
Welcome to the MariaDB monitor.  Commands end with ; or \g.
Your MariaDB connection id is 456
Server version: 10.1.31-MariaDB MariaDB Server

Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
+--------------------+
3 rows in set (0.00 sec)

MariaDB [(none)]>