OpenShift_038：一键部署 hadoop 主从集群

环境：OCP 3.4

本实验全部材料来自 KiwenLau 和我的同事陈耿，在此表示感谢！
KiwenLau 制作的 hadoop cluster 镜像非常棒！

1. 下载 kiwenlau/hadoop-master:0.1.0 镜像
docker pull kiwenlau/hadoop-master:0.1.0
docker pull kiwenlau/hadoop-slave:0.1.0

2. 制作镜像
git clone https://github.com/nichochen/openshift3-demo-hadoop

cd openshift3-demo-hadoop/nic-hadoop-master/
make

vim Dockerfile
Dockerfile 内容如下：
FROM kiwenlau/hadoop-master:0.1.0

# move all confugration files into container
ADD files/* /root/
RUN chmod +x /root/*.sh

EXPOSE 22 7373 7946 9000 50010 50020 50070 50075 50090 50475 8030 8031 8032 8033 8040 8042 8060 8088 50060

3. 下载 nic-hadoop-master 镜像（在 MAC 机器上操作）
docker save -o nic-hadoop-master.tar.gz nic-hadoop-master
scp nic-hadoop-master.tar.gz root@registry.example.com:/opt/ose/images/

docker save -o hadoop-slave.tar.gz kiwenlau/hadoop-slave:0.1.0 
scp hadoop-slave.tar.gz root@registry.example.com:/opt/ose/images/

4. 加载 nic-hadoop-master 镜像（在 Registry 机器上操作）
docker load -i nic-hadoop-master.tar.gz
docker tag nic-hadoop-master:latest registry.example.com:5000/nic-hadoop-master:latest
docker push registry.example.com:5000/nic-hadoop-master

docker load -i hadoop-slave.tar.gz
docker tag kiwenlau/hadoop-slave:0.1.0 registry.example.com:5000/kiwenlau/hadoop-slave:0.1.0
docker push registry.example.com:5000/kiwenlau/hadoop-slave:0.1.0 

5. 创建 template

5.1 创建 hadoop-master template
oc create -f hadoop-master.json -n openshift
hadoop-master.json 内容如下：
{
  "kind": "Template",
  "apiVersion": "v1",
  "metadata": {
    "name": "hadoop-master",
    "creationTimestamp": null,
    "annotations": {
      "description": "Hadoop Master",
      "iconClass": "icon-mysql-database",
      "tags": "hadoop"
    }
  },
  "objects": [
          {
      "kind": "Service",
      "apiVersion": "v1",
      "metadata": {
        "name": "${APP_SERVICE_NAME}",
        "creationTimestamp": null
      },
      "spec": {
        "ports": [
          {
            "name": "app1",
            "protocol": "TCP",
            "port": 22,
            "targetPort": 22,
            "nodePort": 0
          },{
            "name": "app112",
            "protocol": "TCP",
            "port": 7373,
            "targetPort": 7373,
            "nodePort": 0
          },{
            "name": "app2",
            "protocol": "TCP",
            "port": 7946,
            "targetPort": 7946,
            "nodePort": 0
          },{
            "name": "app3",
            "protocol": "TCP",
            "port": 9000,
            "targetPort": 9000,
            "nodePort": 0
          },{
            "name": "app4",
            "protocol": "TCP",
            "port": 50010,
            "targetPort": 50010,
            "nodePort": 0
          },{
            "name": "app5",
            "protocol": "TCP",
            "port": 50020,
            "targetPort": 50020,
            "nodePort": 0
          },{
            "name": "app6",
            "protocol": "TCP",
            "port": 50070,
            "targetPort": 50070,
            "nodePort": 0
          },{
            "name": "app7",
            "protocol": "TCP",
            "port": 50075,
            "targetPort": 50075,
            "nodePort": 0
          },{
            "name": "app8",
            "protocol": "TCP",
            "port": 50090,
            "targetPort": 50090,
            "nodePort": 0
          },{
            "name": "app9",
            "protocol": "TCP",
            "port": 50475,
            "targetPort": 50475,
            "nodePort": 0
          },{
            "name": "app10",
            "protocol": "TCP",
            "port": 8030,
            "targetPort": 8030,
            "nodePort": 0
          },{
            "name": "app11",
            "protocol": "TCP",
            "port": 8031,
            "targetPort": 8031,
            "nodePort": 0
          },{
            "name": "app12",
            "protocol": "TCP",
            "port": 8032,
            "targetPort": 8032,
            "nodePort": 0
          },{
            "name": "app13",
            "protocol": "TCP",
            "port": 8033,
            "targetPort": 8033,
            "nodePort": 0
          },{
            "name": "app14",
            "protocol": "TCP",
            "port": 8040,
            "targetPort": 8040,
            "nodePort": 0
          },{
            "name": "app15",
            "protocol": "TCP",
            "port": 8042,
            "targetPort": 8042,
            "nodePort": 0
          },{
            "name": "app16",
            "protocol": "TCP",
            "port": 8060,
            "targetPort": 8060,
            "nodePort": 0
          },{
            "name": "app17",
            "protocol": "TCP",
            "port": 8088,
            "targetPort": 8088,
            "nodePort": 0
          },{
            "name": "app18",
            "protocol": "TCP",
            "port": 50060,
            "targetPort": 50060,
            "nodePort": 0
          }
        ],
        "selector": {
          "name": "${APP_SERVICE_NAME}"
        },
        "portalIP": "",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    {
      "kind": "DeploymentConfig",
      "apiVersion": "v1",
      "metadata": {
        "name": "${APP_SERVICE_NAME}",
        "creationTimestamp": null
      },
      "spec": {
        "strategy": {
          "type": "Recreate",
          "resources": {}
        },
        "triggers": [
                    {
            "type": "ConfigChange"
          }
        ],
        "replicas": 1,
        "selector": {
          "name": "${APP_SERVICE_NAME}"
        },
        "template": {
          "metadata": {
            "creationTimestamp": null,
            "labels": {
              "name": "${APP_SERVICE_NAME}"
            }
          },
          "spec": {
            "containers": [
              {
                "name": "hadoop-master",
                "image": "nic-hadoop-master",
                "command": ["bash","-c","/root/start-ssh-serf.sh && sleep 365d"],
                "ports": [
                  {
                    "containerPort": 22,
                    "protocol": "TCP"
                  },{
                    "containerPort": 7373,
                    "protocol": "TCP"
                  },{
                    "containerPort": 7946,
                    "protocol": "TCP"
                  },{
                    "containerPort": 9000,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50010,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50020,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50070,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50075,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50090,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50475,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8030,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8031,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8032,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8033,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8040,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8042,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8060,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8088,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50060,
                    "protocol": "TCP"
                  }
                ],
                "env": [
                    {
                    "name": "WAR_URI",
                    "value": "${WAR_URI}"
                  } ],
                "resources": {},
                "volumeMounts": [
                                 ],
                "terminationMessagePath": "/dev/termination-log",
                "imagePullPolicy": "IfNotPresent",
                "capabilities": {},
                "securityContext": {
                  "capabilities": {},
                  "privileged": false
                }
              }
            ],
            "volumes": [
              {
                "name": "${APP_SERVICE_NAME}-data",
                "emptyDir": {
                  "medium": ""
                }
              }
            ],
            "restartPolicy": "Always",
            "dnsPolicy": "ClusterFirst"
          }
        }
      },
      "status": {}
    }
  ],
  "parameters": [
    {
      "name": "APP_SERVICE_NAME",
      "description": "Application service name",
      "value": "hadoop-master",
      "required": true
    }
  ],
  "labels": {
    "template": "app-template"
  }
}

5.2 创建 hadoop-master template
oc create -f hadoop-slave.json -n openshift

hadoop-slave.json 内容如下：
{
  "kind": "Template",
  "apiVersion": "v1",
  "metadata": {
    "name": "hadoop-slave",
    "creationTimestamp": null,
    "annotations": {
      "description": "Hadoop Slave",
      "iconClass": "icon-mysql-database",
      "tags": "hadoop"
    }
  },
  "objects": [
      {
      "kind": "Service",
      "apiVersion": "v1",
      "metadata": {
        "name": "${APP_SERVICE_NAME}",
        "creationTimestamp": null
      },
      "spec": {
        "ports": [
          {
            "name": "app1",
            "protocol": "TCP",
            "port": 22,
            "targetPort": 7373,
            "nodePort": 0
          },{
            "name": "app2",
            "protocol": "TCP",
            "port": 7946,
            "targetPort": 7946,
            "nodePort": 0
          },{
            "name": "app3",
            "protocol": "TCP",
            "port": 9000,
            "targetPort": 9000,
            "nodePort": 0
          },{
            "name": "app4",
            "protocol": "TCP",
            "port": 50010,
            "targetPort": 50010,
            "nodePort": 0
          },{
            "name": "app5",
            "protocol": "TCP",
            "port": 50020,
            "targetPort": 50020,
            "nodePort": 0
          },{
            "name": "app6",
            "protocol": "TCP",
            "port": 50070,
            "targetPort": 50070,
            "nodePort": 0
          },{
            "name": "app7",
            "protocol": "TCP",
            "port": 50075,
            "targetPort": 50075,
            "nodePort": 0
          },{
            "name": "app8",
            "protocol": "TCP",
            "port": 50090,
            "targetPort": 50090,
            "nodePort": 0
          },{
            "name": "app9",
            "protocol": "TCP",
            "port": 50475,
            "targetPort": 50475,
            "nodePort": 0
          },{
            "name": "app10",
            "protocol": "TCP",
            "port": 8030,
            "targetPort": 8030,
            "nodePort": 0
          },{
            "name": "app11",
            "protocol": "TCP",
            "port": 8031,
            "targetPort": 8031,
            "nodePort": 0
          },{
            "name": "app12",
            "protocol": "TCP",
            "port": 8032,
            "targetPort": 8032,
            "nodePort": 0
          },{
            "name": "app13",
            "protocol": "TCP",
            "port": 8033,
            "targetPort": 8033,
            "nodePort": 0
          },{
            "name": "app14",
            "protocol": "TCP",
            "port": 8040,
            "targetPort": 8040,
            "nodePort": 0
          },{
            "name": "app15",
            "protocol": "TCP",
            "port": 8042,
            "targetPort": 8042,
            "nodePort": 0
          },{
            "name": "app16",
            "protocol": "TCP",
            "port": 8060,
            "targetPort": 8060,
            "nodePort": 0
          },{
            "name": "app17",
            "protocol": "TCP",
            "port": 8088,
            "targetPort": 8088,
            "nodePort": 0
          },{
            "name": "app18",
            "protocol": "TCP",
            "port": 50060,
            "targetPort": 50060,
            "nodePort": 0
          }
        ],
        "selector": {
          "name": "${APP_SERVICE_NAME}"
        },
        "portalIP": "",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    {
      "kind": "DeploymentConfig",
      "apiVersion": "v1",
      "metadata": {
        "name": "${APP_SERVICE_NAME}",
        "creationTimestamp": null
      },
      "spec": {
        "strategy": {
          "type": "Recreate",
          "resources": {}
        },
        "triggers": [
                    {
            "type": "ConfigChange"
          }
        ],
        "replicas": 1,
        "selector": {
          "name": "${APP_SERVICE_NAME}"
        },
        "template": {
          "metadata": {
            "creationTimestamp": null,
            "labels": {
              "name": "${APP_SERVICE_NAME}"
            }
          },
          "spec": {
            "containers": [
              {
                "name": "hadoop-slave",
                "image": "kiwenlau/hadoop-slave:0.1.0",
                "command": ["bash","-c","export JOIN_IP=$HADOOP_MASTER_SERVICE_HOST;/root/start-ssh-serf.sh ; ssh -o StrictHostKeyChecking=no $JOIN_IP \"/root/config.sh;/root/restart.sh\"; sleep 365d"],
                "ports": [
                  {
                    "containerPort": 22,
                    "protocol": "TCP"
                  },{
                    "containerPort": 7373,
                    "protocol": "TCP"
                  },{
                    "containerPort": 7946,
                    "protocol": "TCP"
                  },{
                    "containerPort": 9000,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50010,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50020,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50070,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50075,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50090,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50475,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8030,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8031,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8032,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8033,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8040,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8042,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8060,
                    "protocol": "TCP"
                  },{
                    "containerPort": 8088,
                    "protocol": "TCP"
                  },{
                    "containerPort": 50060,
                    "protocol": "TCP"
                  }
                ],
                "env": [
                  ],
                "resources": {},
                "volumeMounts": [
                                 ],
                "terminationMessagePath": "/dev/termination-log",
                "imagePullPolicy": "IfNotPresent",
                "capabilities": {},
                "securityContext": {
                  "capabilities": {},
                  "privileged": false
                }
              }
            ],
            "volumes": [
              {
                "name": "${APP_SERVICE_NAME}-data",
                "emptyDir": {
                  "medium": ""
                }
              }
            ],
            "restartPolicy": "Always",
            "dnsPolicy": "ClusterFirst"
          }
        }
      },
      "status": {}
    }
  ],
  "parameters": [
    {
      "name": "APP_SERVICE_NAME",
      "description": "Application service name",
      "value": "hadoop-slave",
      "required": true
    }
  ],
  "labels": {
    "template": "hadoop-slave-template"
  }
}


6.  创建应用

6.1 选择 hadoop-master template，创建应用
oc new-app --template=hadoop-master

6.2 选择 hadoop-slave template，创建应用
oc new-app --template=hadoop-slave

 
7.  进入 hadoop-master pod，执行
bash
serf members
cd
./start-hadoop.sh

8.  清除 hadoop 主从集群（在 Master 机器上操作）
oc delete svc $(oc get svc | grep hadoop |awk '{print $1}')
oc delete dc $(oc get dc | grep hadoop |awk '{print $1}')
oc delete rc $(oc get rc | grep hadoop |awk '{print $1}')
oc delete routes $(oc get routes | grep hadoop |awk '{print $1}')
oc delete pod $(oc get pod | grep hadoop |awk '{print $1}')

参考文献：
1. https://github.com/nichochen/openshift3-demo-hadoop
2. http://kiwenlau.com/2016/06/12/160612-hadoop-cluster-docker-update/