OpenShift_037：一键部署 mysql 双活集群

环境：OCP 3.4

本实验全部材料来自我的同事陈耿和王洪涛，在此向他们表示感谢！

1. 部署架构图 

2. 制作镜像

git clone https://github.com/nichochen/openshift3-demo-mysql-mm.git

2.1 制作 mysql 双活镜像
cd ~/mygit/openshift3-demo-mysql-mm
vim Images/nic-mysql-mm/Dockerfile
修改后，内容如下：
#FROM  openshift/mysql-55-rhel7
FROM  registry.access.redhat.com/rhscl/mysql-56-rhel7
USER root
# move all confugration files into container
ADD files/* /usr/share/container-scripts/mysql/
RUN chmod +x /usr/share/container-scripts/mysql/*.sh
RUN chmod +x /usr/share/container-scripts/mysql/run*
RUN mv  /usr/share/container-scripts/mysql/run-mysqld-mm /usr/bin/
USER mysql

EXPOSE 3306

编译生成镜像
cd ~/mygit/openshift3-demo-mysql-mm/Images/nic-mysql-mm
make

2.2 制作 mysql 前端分发器镜像
cd ~/mygit/openshift3-demo-mysql-mm
vim Images/nic-haproxy-mysql-mm/Dockerfile
内容如下：
FROM haproxy:latest
RUN useradd haproxy
COPY files/haproxy.cfg.template /usr/local/etc/haproxy/haproxy.cfg.template
COPY files/run-haproxy.sh    /usr/bin/run-haproxy.sh
RUN  chmod +x /usr/bin/run-haproxy.sh;chown -R haproxy /usr/local/etc/haproxy
USER haproxy

CMD run-haproxy.sh
EXPOSE 3306 8888

编译生成镜像
cd ~/mygit/openshift3-demo-mysql-mm/Images/nic-haproxy-mysql-mm
make

3. 下载 nic-mysql-56-rhel7-mm 和 nic-haproxy-mysql-mm 镜像（在 MAC 机器上操作）
docker save -o nic-mysql-56-rhel7-mm.tar.gz nic-mysql-56-rhel7-mm
scp nic-mysql-56-rhel7-mm.tar.gz root@registry.example.com:/opt/ose/images/

docker save -o nic-haproxy-mysql-mm.tar.gz nic-haproxy-mysql-mm
scp nic-haproxy-mysql-mm.tar.gz root@registry.example.com:/opt/ose/images/

4. 加载 nic-mysql-56-rhel7-mm 和 nic-haproxy-mysql-mm 镜像（在 Registry 机器上操作）
cd /opt/ose/images

docker load -i nic-mysql-56-rhel7-mm.tar.gz
docker tag nic-mysql-56-rhel7-mm:latest registry.example.com:5000/nic-mysql-56-rhel7-mm:latest
docker push registry.example.com:5000/nic-mysql-56-rhel7-mm

docker load -i nic-haproxy-mysql-mm.tar.gz
docker tag nic-haproxy-mysql-mm:latest registry.example.com:5000/nic-haproxy-mysql-mm:latest
docker push registry.example.com:5000/nic-haproxy-mysql-mm

5. 创建 NFS Server (在 Registry 机器上操作)
yum install –y nfs-utils

export volname=mysql-master-a
mkdir -p /srv/nfs/${volname}
chown nfsnobody:nfsnobody /srv/nfs/${volname}
chmod 700 /srv/nfs/${volname}
echo "/srv/nfs/${volname} *(rw,sync,all_squash)" >> /etc/exports

export volname=mysql-master-b
mkdir -p /srv/nfs/${volname}
chown nfsnobody:nfsnobody /srv/nfs/${volname}
chmod 700 /srv/nfs/${volname}
echo "/srv/nfs/${volname} *(rw,sync,all_squash)" >> /etc/exports

systemctl enable nfs-server
systemctl restart rpcbind
systemctl restart nfs-server nfs-lock nfs-idmap

6. 创建 PV (在 Master 机器上操作)

6.1 创建 mysql-master-a-pv
oc create -f mysql-master-a-pv.json

mysql-master-a-pv.json 内容如下：
{
  "apiVersion": "v1",
  "kind": "PersistentVolume",
  "metadata": {
    "name": "mysql-master-a"
  },
  "spec": {
    "capacity": {
        "storage": "512Mi"
    },
    "accessModes": [ "ReadWriteMany" ],
    "nfs": {
        "path": "/srv/nfs/mysql-master-a",
        "server": "registry.example.com"
    },
    "persistentVolumeReclaimPolicy": "Retain"
  }
}

6.2 创建 mysql-master-a-pv
oc create -f mysql-master-b-pv.json

mysql-master-b-pv.json 内容如下：
{
  "apiVersion": "v1",
  "kind": "PersistentVolume",
  "metadata": {
    "name": "mysql-master-b"
  },
  "spec": {
    "capacity": {
        "storage": "512Mi"
    },
    "accessModes": [ "ReadWriteMany" ],
    "nfs": {
        "path": "/srv/nfs/mysql-master-b",
        "server": "registry.example.com"
    },
    "persistentVolumeReclaimPolicy": "Retain"
  }
}

7. 创建 mysql-master-master template (在 Master 机器上操作)
oc create -f mysql-master-master-template.json -n openshift

mysql-master-master-template.json 内容如下：
{
  "kind": "Template",
  "apiVersion": "v1",
  "metadata": {
    "name": "mysql-mm-replication-example",
    "creationTimestamp": null,
    "annotations": {
      "description": "MySQL MM Replication Example",
      "iconClass": "icon-database",
      "tags": "database,mysql,replication"
    }
  },
  "parameters": [
    {
      "name": "MYSQL_MASTER_SERVICE_NAME",
      "description": "Service name for MySQL master service",
      "value": "mysql-master"
    },
    {
      "name": "MYSQL_MASTER_USER",
      "description": "The username used for master-slave replication",
      "value": "master"
    },
    {
      "name": "MYSQL_MASTER_PASSWORD",
      "description": "The password for the MySQL master user",
      "generate": "expression",
      "from": "[a-zA-Z0-9]{12}"
    },
    {
      "name": "MYSQL_USER",
      "description": "The username that clients will use to connect to MySQL server",
      "value": "user"
    },
    {
      "name": "MYSQL_PASSWORD",
      "description": "The password for the MySQL master user",
      "generate": "expression",
      "from": "[a-zA-Z0-9]{12}"
    },
    {
      "name": "MYSQL_DATABASE",
      "description": "The name of the database that will be created",
      "value": "userdb"
    },
    {
      "name": "MYSQL_ROOT_PASSWORD",
      "description": "The password for the MySQL adminitrator",
      "generate": "expression",
      "from": "[a-zA-Z0-9]{12}"
    }
  ],
  "objects": [
    {
      "kind": "PersistentVolumeClaim",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-master-a"
      },
      "spec": {
        "accessModes": [
          "ReadWriteMany"
        ],
        "resources": {
          "requests": {
            "storage": "512Mi"
          }
        }
      }
    },{
      "kind": "PersistentVolumeClaim",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-master-b"
      },
      "spec": {
        "accessModes": [
          "ReadWriteMany"
        ],
        "resources": {
          "requests": {
            "storage": "512Mi"
          }
        }
      }
    },
    {
      "kind": "Service",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-master-a",
        "labels": {
          "name": "mysql-master-a"
        }
      },
      "spec": {
        "ports": [
          {
            "protocol": "TCP",
            "port": 3306,
            "targetPort": 3306,
            "nodePort": 0
          }
        ],
        "selector": {
          "name": "mysql-master-a"
        },
        "portalIP": "",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    {
      "kind": "Service",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-master-b",
        "labels": {
          "name": "mysql-master-b"
        }
      },
      "spec": {
        "ports": [
          {
            "protocol": "TCP",
            "port": 3306,
            "targetPort": 3306,
            "nodePort": 0
          }
        ],
        "selector": {
          "name": "mysql-master-b"
        },
        "portalIP": "",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    {
      "kind": "DeploymentConfig",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-master-a",
        "creationTimestamp": null
      },
      "spec": {
        "strategy": {
          "type": "Recreate",
          "resources": {}
        },
        "triggers": [
          {
            "type": "ConfigChange"
          }
        ],
        "replicas": 1,
        "selector": {
          "name": "mysql-master-a"
        },
        "template": {
          "metadata": {
            "creationTimestamp": null,
            "labels": {
              "name": "mysql-master-a"
            }
          },
          "spec": {
            "volumes": [
               {
                "name": "mysql-master-a-data",
                "persistentVolumeClaim": {
                  "claimName": "mysql-master-a"
                }
              }
            ],
            "containers": [
              {
                "name": "server",
                "image": "nic-mysql-56-rhel7-mm",
                "args": [
                  "run-mysqld-mm"
                ],
                "ports": [
                  {
                    "containerPort": 3306,
                    "protocol": "TCP"
                  }
                ],
                "env": [
                    {
                    "name": "MYSQL_TARGET_MASTER_SERVICE_NAME",
                    "value": "MYSQL_MASTER_B_SERVICE_HOST"
                  },{
                    "name": "AUTO_INCREMENT_OFFSET",
                    "value": "1"
                  },
                  {
                    "name": "MYSQL_MASTER_USER",
                    "value": "${MYSQL_MASTER_USER}"
                  },
                  {
                    "name": "MYSQL_MASTER_PASSWORD",
                    "value": "${MYSQL_MASTER_PASSWORD}"
                  },
                  {
                    "name": "MYSQL_USER",
                    "value": "${MYSQL_USER}"
                  },
                  {
                    "name": "MYSQL_PASSWORD",
                    "value": "${MYSQL_PASSWORD}"
                  },
                  {
                    "name": "MYSQL_DATABASE",
                    "value": "${MYSQL_DATABASE}"
                  },
                  {
                    "name": "MYSQL_ROOT_PASSWORD",
                    "value": "${MYSQL_ROOT_PASSWORD}"
                  }
                ],
                "volumeMounts": [
                  {
                    "name": "mysql-master-a-data",
                    "mountPath": "/var/lib/mysql/data"
                  }
                ],
                "resources": {},
                "terminationMessagePath": "/dev/termination-log",
                "imagePullPolicy": "IfNotPresent",
                "securityContext": {
                  "capabilities": {},
                  "privileged": false
                }
              }
            ],
            "restartPolicy": "Always",
            "dnsPolicy": "ClusterFirst"
          }
        }
      },
      "status": {}
    },
    {
      "kind": "DeploymentConfig",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-master-b",
        "creationTimestamp": null
      },
      "spec": {
        "strategy": {
          "type": "Recreate",
          "resources": {}
        },
        "triggers": [
          {
            "type": "ConfigChange"
          }
        ],
        "replicas": 1,
        "selector": {
          "name": "mysql-master-b"
        },
        "template": {
          "metadata": {
            "creationTimestamp": null,
            "labels": {
              "name": "mysql-master-b"
            }
          },
          "spec": {
            "volumes": [
               {
                "name": "mysql-master-b-data",
                "persistentVolumeClaim": {
                  "claimName": "mysql-master-b"
                }
              }
            ],
            "containers": [
              {
                "name": "server",
                "image": "nic-mysql-56-rhel7-mm",
                "args": [
                  "run-mysqld-mm"
                ],
                "ports": [
                  {
                    "containerPort": 3306,
                    "protocol": "TCP"
                  }
                ],
                "env": [
              {
                    "name": "MYSQL_TARGET_MASTER_SERVICE_NAME",
                    "value": "MYSQL_MASTER_A_SERVICE_HOST"
                  },{
                    "name": "AUTO_INCREMENT_OFFSET",
                    "value": "2"
                  },
                  {
                    "name": "MYSQL_MASTER_SERVICE_NAME",
                    "value": "mysql-master-a"
                  },
                  {
                    "name": "MYSQL_MASTER_USER",
                    "value": "${MYSQL_MASTER_USER}"
                  },
                  {
                    "name": "MYSQL_MASTER_PASSWORD",
                    "value": "${MYSQL_MASTER_PASSWORD}"
                  },
                  {
                    "name": "MYSQL_DATABASE",
                    "value": "userdb2"
                  }
                ],
                "volumeMounts": [
                  {
                    "name": "mysql-master-b-data",
                    "mountPath": "/var/lib/mysql/data"
                  }
                ],
                "resources": {},
                "terminationMessagePath": "/dev/termination-log",
                "imagePullPolicy": "IfNotPresent",
                "securityContext": {
                  "capabilities": {},
                  "privileged": false
                }
              }
            ],
            "restartPolicy": "Always",
            "dnsPolicy": "ClusterFirst"
          }
        }
      },
      "status": {}
    }
  ]
}

8. 创建 mysql-haproxy template (在 Master 机器上操作)
oc create -f mysql-haproxy-template.json -n openshift

mysql-haproxy-template.json 内容如下：
{
  "kind": "Template",
  "apiVersion": "v1",
  "metadata": {
    "name": "mysql-haproxy",
    "creationTimestamp": null,
    "annotations": {
      "description": "MySQL HAProxy",
      "iconClass": "icon-database",
      "tags": "database,mysql,replication"
    }
  },
  "objects": [
{
    "kind": "Route",
    "apiVersion": "v1",
    "metadata": {
        "name": "mysql-haproxy",
        "creationTimestamp": null,
        "labels": {
            "name": "mysql-haproxy"
        }
    },
    "spec": {
        "host": "mysql-status.applications.apps.example.com",
        "to": {
            "kind": "Service",
            "name": "mysql-haproxy"
        },
        "port": {
            "targetPort": "8888"
        }
    },
    "status": {}
},
    {
      "kind": "Service",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-haproxy",
        "labels": {
          "name": "mysql-haproxy"
        }
      },
      "spec": {
        "ports": [
          {
            "name": "mysql",
            "protocol": "TCP",
            "port": 3306,
            "targetPort": 3306,
            "nodePort": 0
          },{
            "name": "status",
            "protocol": "TCP",
            "port": 8888,
            "targetPort": 8888,
            "nodePort": 0
          }
        ],
        "selector": {
          "name": "mysql-haproxy"
        },
        "portalIP": "",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    {
      "kind": "DeploymentConfig",
      "apiVersion": "v1",
      "metadata": {
        "name": "mysql-haproxy",
        "creationTimestamp": null
      },
      "spec": {
        "strategy": {
          "type": "Recreate",
          "resources": {}
        },
        "triggers": [
          {
            "type": "ConfigChange"
          }
        ],
        "replicas": 1,
        "selector": {
          "name": "mysql-haproxy"
        },
        "template": {
          "metadata": {
            "creationTimestamp": null,
            "labels": {
              "name": "mysql-haproxy"
            }
          },
          "spec": {
            "containers": [
              {
                "name": "haproxy",
                "image": "nic-haproxy-mysql-mm",
                "ports": [
                  {
                    "containerPort": 3306,
                    "protocol": "TCP"
                  },
                  {
                    "containerPort": 8888,
                    "protocol": "TCP"
                  }
                ],
                "env": [

                ],
                "resources": {},
                "terminationMessagePath": "/dev/termination-log",
                "imagePullPolicy": "IfNotPresent",
                "securityContext": {
                  "capabilities": {},
                  "privileged": false
                }
              }
            ],
            "restartPolicy": "Always",
            "dnsPolicy": "ClusterFirst"
          }
        }
      },
      "status": {}
    }
  ]
}

9. 创建 mysql 双活应用
选择 mysql-mm-replication-example 模板，创建应用。

选择 mysql-haproxy 模板，创建应用。

访问 http://mysql-status.applications.apps.example.com/dbs admin/admin

访问 http://mywebsql-applications.apps.example.com，使用 mysql-haproxy service IP 地址作为参数。
创建表，插入数据，发现 mysql-master-a 和 mysql-master-b 都有数据。

10. 清除 mysql 双活集群

10.1 在 Master 机器上操作
oc delete svc $(oc get svc | grep mysql |awk '{print $1}')
oc delete dc $(oc get dc | grep mysql |awk '{print $1}')
oc delete rc $(oc get rc | grep mysql |awk '{print $1}')
oc delete routes $(oc get routes | grep mysql |awk '{print $1}')
oc delete pod $(oc get pod | grep mysql |awk '{print $1}')
oc delete pvc mysql-master-a
oc delete pvc mysql-master-b
oc delete pv mysql-master-a
oc delete pv mysql-master-b

10.2 在 Registry 机器上操作
cd /srv/nfs/mysql-master-a
rm -rf *

cd /srv/nfs/mysql-master-b
rm -rf *

参考文献：
1. https://github.com/nichochen/openshift3-demo-mysql-mm