2017年5月23日星期二

OpenShift_082:离线安装 OCP 3.5 之安装后配置

1. 配置用户(在 Master 上操作)
安装 httpd-tools
yum -y install httpd-tools;
创建一个 admin 用户,并且赋予权限
htpasswd -cb /etc/origin/master/htpasswd admin redhat;
oadm policy add-cluster-role-to-user admin admin

2. 配置二级域名(在 Master 上操作)
cp /etc/origin/master/master-config.yaml /etc/origin/master/master-config.yaml.bak.$(date "+%Y%m%d%H%M%S");
sed -i 's/.*subdomain.*/  subdomain: apps.example.com/' /etc/origin/master/master-config.yaml;

3. 重启 OpenShift,使上述修改生效(在 Master 上操作)
systemctl restart atomic-openshift-{master,node};

4. 如果之前自动安装 Registry Console 失败,以下手动重新安装

4.1 之前自动安装 Registry Console 有错,执行以下命令清除
oc delete dc registry-console
oc delete svc registry-console
oc delete route registry-console
oc delete is registry-console
oc delete oauthclients cockpit-oauth-client

4.2 创建 registry-console route
oc create route passthrough --service registry-console --port registry-console -n default

4.3 部署 registry console 应用
oc new-app -n default --template=registry-console \
    -p IMAGE_PREFIX="registry.example.com:5000/openshift3/" \
    -p IMAGE_VERSION="3.5" \
    -p OPENSHIFT_OAUTH_PROVIDER_URL="https://master.example.com:8443" \
    -p REGISTRY_HOST=$(oc get route docker-registry -n default --template='{{ .spec.host }}') \
    -p COCKPIT_KUBE_URL=$(oc get route registry-console -n default --template='https://{{ .spec.host }}')

5. 添加 Image Stream(在 Master 上操作)

5.1 查看当前的 Image Stream 
oc get is -n openshift
输出如下:
NAME                                  DOCKER REPO                                                                    TAGS                      UPDATED
dotnet                                172.30.132.67:5000/openshift/dotnet                                            1.1,1.0                
fis-java-openshift                    registry.access.redhat.com/jboss-fuse-6/fis-java-openshift                                            
fis-karaf-openshift                   registry.access.redhat.com/jboss-fuse-6/fis-karaf-openshift                                            
jboss-amq-62                          registry.access.redhat.com/jboss-amq-6/amq62-openshift                                                
jboss-datagrid65-openshift            registry.access.redhat.com/jboss-datagrid-6/datagrid65-openshift                                      
jboss-datavirt63-openshift            registry.access.redhat.com/jboss-datavirt-6/datavirt63-openshift                                      
jboss-decisionserver62-openshift      registry.access.redhat.com/jboss-decisionserver-6/decisionserver62-openshift                          
jboss-decisionserver63-openshift      registry.access.redhat.com/jboss-decisionserver-6/decisionserver63-openshift                          
jboss-eap64-openshift                 registry.access.redhat.com/jboss-eap-6/eap64-openshift                                                
jboss-eap70-openshift                 registry.access.redhat.com/jboss-eap-7/eap70-openshift                                                
jboss-processserver63-openshift       registry.access.redhat.com/jboss-processserver-6/processserver63-openshift                            
jboss-webserver30-tomcat7-openshift   registry.access.redhat.com/jboss-webserver-3/webserver30-tomcat7-openshift                            
jboss-webserver30-tomcat8-openshift   registry.access.redhat.com/jboss-webserver-3/webserver30-tomcat8-openshift                            
jenkins                               172.30.132.67:5000/openshift/jenkins                                           2,1                    
mariadb                               172.30.132.67:5000/openshift/mariadb                                           10.1                    
mongodb                               172.30.132.67:5000/openshift/mongodb                                           3.2,2.6,2.4            
mysql                                 172.30.132.67:5000/openshift/mysql                                             5.7,5.6,5.5            
nodejs                                172.30.132.67:5000/openshift/nodejs                                            0.10,4                  
perl                                  172.30.132.67:5000/openshift/perl                                              5.24,5.20,5.16          
php                                   172.30.132.67:5000/openshift/php                                               7.0,5.6,5.5            
postgresql                            172.30.132.67:5000/openshift/postgresql                                        9.4,9.2,9.5            
python                                172.30.132.67:5000/openshift/python                                            3.5,3.4,3.3 + 1 more...
redhat-openjdk18-openshift            registry.access.redhat.com/redhat-openjdk-18/openjdk18-openshift                                      
redhat-sso70-openshift                registry.access.redhat.com/redhat-sso-7/sso70-openshift                                                
redis                                 172.30.132.67:5000/openshift/redis                                             3.2                    
ruby                                  172.30.132.67:5000/openshift/ruby                                              2.3,2.2,2.0            

5.2 删除原有的 Image Stream
for i in $(oc get is -n openshift --no-headers|awk '{print $1}') ; do oc delete is $i -n openshift; done;

5.3 创建指向本地 Docker Registry 的 Image Stream

REDHAT_REG="registry.access.redhat.com";
PRIVATE_REG="registry.example.com:5000";

sed s/"${REDHAT_REG}"/"${PRIVATE_REG}"/g  /usr/share/openshift/examples/image-streams/image-streams-rhel7.json |sed '/"creationTimestamp": null/a\\t,"annotations": {"openshift.io/image.insecureRepository": "true"}' |oc create -n openshift -f - ;

sed s/"${REDHAT_REG}"/"${PRIVATE_REG}"/g /usr/share/openshift/examples/xpaas-streams/jboss-image-streams.json |sed '/"creationTimestamp": null/a\\t,"annotations": {"openshift.io/image.insecureRepository": "true"}' |oc create -n openshift -f - ;

sed s/"${REDHAT_REG}"/"${PRIVATE_REG}"/g /usr/share/openshift/examples/image-streams/dotnet_imagestreams.json |sed '/"creationTimestamp": null/a\\t,"annotations": {"openshift.io/image.insecureRepository": "true"}' |oc create -n openshift -f - ;

5.4 再次查看 Image Stream
oc get is -n openshift
输出如下:
NAME                                  DOCKER REPO                                                                   TAGS                      UPDATED
dotnet                                172.30.132.67:5000/openshift/dotnet                                           1.0,1.1              
jboss-amq-62                          registry.example.com:5000/jboss-amq-6/amq62-openshift                                                
jboss-datagrid65-openshift            registry.example.com:5000/jboss-datagrid-6/datagrid65-openshift                                      
jboss-datavirt63-openshift            registry.example.com:5000/jboss-datavirt-6/datavirt63-openshift                                      
jboss-decisionserver62-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver62-openshift                          
jboss-decisionserver63-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift                          
jboss-eap64-openshift                 registry.example.com:5000/jboss-eap-6/eap64-openshift                                                
jboss-eap70-openshift                 registry.example.com:5000/jboss-eap-7/eap70-openshift                                                
jboss-processserver63-openshift       registry.example.com:5000/jboss-processserver-6/processserver63-openshift                            
jboss-webserver30-tomcat7-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat7-openshift                            
jboss-webserver30-tomcat8-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat8-openshift                            
jenkins                               172.30.132.67:5000/openshift/jenkins                                          1,2                    
mariadb                               172.30.132.67:5000/openshift/mariadb                                          10.1                    
mongodb                               172.30.132.67:5000/openshift/mongodb                                          3.2,2.6,2.4            
mysql                                 172.30.132.67:5000/openshift/mysql                                            5.5,5.7,5.6            
nodejs                                172.30.132.67:5000/openshift/nodejs                                           4,0.10                  
perl                                  172.30.132.67:5000/openshift/perl                                             5.24,5.20,5.16          
php                                   172.30.132.67:5000/openshift/php                                              5.6,5.5,7.0            
postgresql                            172.30.132.67:5000/openshift/postgresql                                       9.5,9.4,9.2            
python                                172.30.132.67:5000/openshift/python                                           3.4,3.3,2.7 + 1 more...
redhat-openjdk18-openshift            registry.example.com:5000/redhat-openjdk-18/openjdk18-openshift                                      
redhat-sso70-openshift                registry.example.com:5000/redhat-sso-7/sso70-openshift                                                
redis                                 172.30.132.67:5000/openshift/redis                                            3.2                    
ruby                                  172.30.132.67:5000/openshift/ruby                                             2.3,2.2,2.0                    

5.5 导入列表中所有 Image
可以看到有些 Image Stream 没有 Tag,需要导入 Image
for i in $(oc get is -n openshift --no-headers|awk '{print $1}'); do oc import-image $i --insecure --all -n openshift;done

注意,不是所有的 image 都能导入成功,这个跟导入的 Image 有关。
如果没有该 Image,是无法成功导入 Image Stream 的。
并且,一个软件有很多版本的 Image,导入了哪个 Image,哪个 Image Stream Tag 才能导入成功。
比如 mysql 这个 Image 我导入了 5.7 和 5.6 版本,在执行 oc import-image 时,会报告 5.7 和 5.6 成功导入,而 5.5 没有导入成功。
oc import-image mysql --insecure -n openshift
输出如下:
The import completed successfully.

Name: mysql
Namespace: openshift
Created: 3 minutes ago
Labels: <none>
Annotations: openshift.io/display-name=MySQL
openshift.io/image.dockerRepositoryCheck=2017-05-24T10:12:33Z
Docker Pull Spec: 172.30.132.67:5000/openshift/mysql
Unique Images: 2
Tags: 4

5.7 (latest)
  tagged from registry.example.com:5000/rhscl/mysql-57-rhel7:latest
    will use insecure HTTPS or HTTP connections

  Provides a MySQL 5.7 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/mysql-container/tree/master/5.7/README.md.
  Tags: mysql

  * registry.example.com:5000/rhscl/mysql-57-rhel7@sha256:bcec20eec2b4292ba3e0ba91e05f07b1dd8f04f4fda2eaf1e32e947f613975a4
      2 minutes ago

5.6
  tagged from registry.example.com:5000/rhscl/mysql-56-rhel7:latest
    will use insecure HTTPS or HTTP connections

  Provides a MySQL 5.6 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/mysql-container/tree/master/5.6/README.md.
  Tags: mysql

  * registry.example.com:5000/rhscl/mysql-56-rhel7@sha256:cd7981a6fab1f9ac08ee60348b9c79badbb32a858ead07198eb5a493cc13c741
      2 minutes ago

5.5
  tagged from registry.example.com:5000/openshift3/mysql-55-rhel7:latest

  Provides a MySQL 5.5 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/mysql-container/tree/master/5.5/README.md.
  Tags: hidden, mysql

  ! error: Import failed (NotFound): dockerimage "registry.example.com:5000/openshift3/mysql-55-rhel7:latest" not found
      2 minutes ago

5.6 再次查看 Image Stream
oc get is -n openshift
输出如下:
NAME                                  DOCKER REPO                                                                   TAGS                         UPDATED
dotnet                                172.30.132.67:5000/openshift/dotnet                                           1.1,1.0                    
jboss-amq-62                          registry.example.com:5000/jboss-amq-6/amq62-openshift                         latest                       18 seconds ago
jboss-datagrid65-openshift            registry.example.com:5000/jboss-datagrid-6/datagrid65-openshift               latest                    
jboss-datavirt63-openshift            registry.example.com:5000/jboss-datavirt-6/datavirt63-openshift               latest                    
jboss-decisionserver62-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver62-openshift   latest                    
jboss-decisionserver63-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift   latest                       17 seconds ago
jboss-eap64-openshift                 registry.example.com:5000/jboss-eap-6/eap64-openshift                         latest                       17 seconds ago
jboss-eap70-openshift                 registry.example.com:5000/jboss-eap-7/eap70-openshift                         latest                       17 seconds ago
jboss-processserver63-openshift       registry.example.com:5000/jboss-processserver-6/processserver63-openshift     latest                       17 seconds ago
jboss-webserver30-tomcat7-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat7-openshift     latest                       16 seconds ago
jboss-webserver30-tomcat8-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat8-openshift     latest                       16 seconds ago
jenkins                               172.30.132.67:5000/openshift/jenkins                                          latest,2,1                   16 seconds ago
mariadb                               172.30.132.67:5000/openshift/mariadb                                          10.1                      
mongodb                               172.30.132.67:5000/openshift/mongodb                                          latest,3.2,2.4 + 1 more...   16 seconds ago
mysql                                 172.30.132.67:5000/openshift/mysql                                            latest,5.7,5.6 + 1 more...   15 seconds ago
nodejs                                172.30.132.67:5000/openshift/nodejs                                           0.10,4                    
perl                                  172.30.132.67:5000/openshift/perl                                             5.24,5.20,5.16            
php                                   172.30.132.67:5000/openshift/php                                              latest,7.0,5.6 + 1 more...   15 seconds ago
postgresql                            172.30.132.67:5000/openshift/postgresql                                       9.4,9.2,9.5                
python                                172.30.132.67:5000/openshift/python                                           latest,3.5,3.4 + 2 more...   15 seconds ago
redhat-openjdk18-openshift            registry.example.com:5000/redhat-openjdk-18/openjdk18-openshift               latest                    
redhat-sso70-openshift                registry.example.com:5000/redhat-sso-7/sso70-openshift                        latest                    
redis                                 172.30.132.67:5000/openshift/redis                                            latest,3.2                   14 seconds ago
ruby                                  172.30.132.67:5000/openshift/ruby                                             latest,2.3,2.2 + 1 more...   14 seconds ago

可以看到有些镜像多了很多版本。

6. 访问 OpenShift 控制台
https://master.example.com:8443 admin/redhat

OpenShift_081:离线安装 OCP 3.5 之安装

1.  SSH 互信配置(在 Master 上操作)
ssh-keygen;

for i in  master.example.com node1.example.com node2.example.com;
do
ssh-copy-id $i;
done;

2. 安装(在 Master 上操作)
cat > /etc/ansible/hosts <<EOF
[OSEv3:children]
masters
nodes

[OSEv3:vars]
ansible_ssh_user=root
deployment_type=openshift-enterprise
openshift_release=v3.5

#oreg_url=registry.example.com:5000/openshift3/ose-${component}:${version}
openshift_docker_additional_registries=registry.example.com:5000
openshift_docker_insecure_registries=registry.example.com:5000
openshift_examples_modify_imagestreams=true
openshift_cockpit_deployer_prefix=registry.example.com:5000/openshift3/

openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]

openshift_clock_enabled=true

openshift_hosted_router_selector="infra=true"
openshift_hosted_registry_selector="infra=true"

#openshift_master_cluster_method=native
#openshift_master_cluster_hostname=master.example.com
#openshift_master_cluster_public_hostname=master.example.com

#openshift_master_default_subdomain=apps.example.com

#openshift_node_kubelet_args={'pods-per-core': ['10'], 'max-pods':['25']}

[masters]
master.example.com

[nodes]
master.example.com
node1.example.com openshift_node_labels="{'infra': 'true'}"

node2.example.com

EOF

说明:
(1)注释 oreg_url=registry.example.com:5000/openshift3/ose-${component}:${version}
经过多次实验,发现这个参数会引起如下错误,因此将其注释:
TASK [openshift_master : Start and enable master] ******************************
FAILED - RETRYING: TASK: openshift_master : Start and enable master (1 retries left).
fatal: [master.example.com]: FAILED! => {
    "attempts": 1,
    "changed": false,
    "failed": true
}

MSG:

Unable to start service atomic-openshift-master: Job for atomic-openshift-master.service failed because the control process exited with error code. See "systemctl status atomic-openshift-master.service" and "journalctl -xe" for details

(2)增加 openshift_cockpit_deployer_prefix=registry.example.com:5000/openshift3/
这样就可以正确 pull 到 registry-console 镜像。如果不加,它会到 registry.access.redhat.com 去 pull 镜像,是无法成功的。

执行安装脚本
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml;

安装大概需要 11 分钟,安装成功,最后会输出:
PLAY RECAP *********************************************************************
localhost                  : ok=8    changed=0    unreachable=0    failed=0
master.example.com         : ok=524  changed=140  unreachable=0    failed=0
node1.example.com          : ok=217  changed=60   unreachable=0    failed=0
node2.example.com          : ok=217  changed=60   unreachable=0    failed=0

如果安装失败,查看错误,修改 hosts 脚本,然后运行以下命令清理环境,再重装
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml

卸载 atomic-openshift-utils(在各个机器上操作)
yum remove atomic-openshift-utils

3. 安装后检查(在 Master 上操作)
oc get node --show-labels
输出如下:
NAME                 STATUS                     AGE       LABELS
master.example.com   Ready,SchedulingDisabled   5m        beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master.example.com
node1.example.com    Ready                      5m        beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node1.example.com
node2.example.com    Ready                      5m        beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node2.example.com

oc get pod -o wide
输出如下:
NAME                       READY     STATUS    RESTARTS   AGE       IP               NODE
docker-registry-1-9v8rt    1/1       Running   0          6m        10.129.0.4       node1.example.com
registry-console-1-n8prf   1/1       Running   0          5m        10.130.0.3       node2.example.com
router-1-9vnv6             1/1       Running   0          6m        192.168.56.113   node1.example.com

这里 registry、router、registry-console 安装成功。
如果 registry-console 安装失败,也可以后面手工安装 registry-console。

至此,说明安装成功!

4. 检查所有机器的 /etc/sysconfig/docker 文件内容 (在 Master、Node1、Node2 上操作)
OpenShift 成功安装后,会重置 /etc/sysconfig/docker文件,为保证后面使用正常,需要把这些文件改成正确的内容。
经过确认,Master、Node1、Node2 上的这个文件都是正确的。
如果不放心,运行以下脚本:
cp /etc/sysconfig/docker /etc/sysconfig/docker.bak.$(date "+%Y%m%d%H%M%S");
sed  -i s/".*OPTIONS=.*"/"OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0\/16 --insecure-registry registry.example.com:5000'"/g /etc/sysconfig/docker;
sed -i 's/registry.access.redhat.com/registry.example.com:5000/g' /etc/sysconfig/docker;
systemctl restart docker

OpenShift_080:离线安装 OCP 3.5 之 环境准备 之 本地 Docker Registry 配置

1.  安装并配置 Docker Registry (在 Registry 上操作)
yum -y install docker-distribution;
systemctl enable docker-distribution;
systemctl start docker-distribution;

mkdir /opt/ose/images

2. 拷贝之前下载的镜像到 Registry(在 MAC 上操作)
scp ose-images-core-20170522.tar.gz root@192.168.56.112:/opt/ose/images/
scp ose-images-logging_metric-20170522.tar.gz root@192.168.56.112:/opt/ose/images/
scp ose-images-apps-latest_20170522.tar.gz root@192.168.56.112:/opt/ose/images/
scp ose-images-apps-extra-latest_20170522.tar.gz root@192.168.56.112:/opt/ose/images/

3. 加载所有镜像(在 Registry 上操作
cd /opt/ose/images/
for i in `ls *.tar.gz` ; do docker load -i $i; done;

docker images;

输出如下:
REPOSITORY                                                                   TAG                 IMAGE ID            CREATED             SIZE
registry.access.redhat.com/openshift3/metrics-hawkular-metrics               v3.5                94cd687068c2        7 days ago          1.269 GB
registry.access.redhat.com/openshift3/metrics-heapster                       v3.5                cc6454e9d765        7 days ago          317.9 MB
registry.access.redhat.com/openshift3/registry-console                       v3.5                e53d505ae98e        7 days ago          476.2 MB
registry.access.redhat.com/openshift3/logging-kibana                         v3.5                f9bfc4c46fb5        7 days ago          342.6 MB
registry.access.redhat.com/openshift3/metrics-cassandra                      v3.5                581c9cad4aaf        7 days ago          539.5 MB
registry.access.redhat.com/openshift3/logging-fluentd                        v3.5                1421c119d952        7 days ago          232.8 MB
registry.access.redhat.com/openshift3/logging-elasticsearch                  v3.5                1dca1f8506c0        7 days ago          399.5 MB
registry.access.redhat.com/openshift3/metrics-deployer                       v3.5                c044fb3618c3        7 days ago          892.7 MB
registry.access.redhat.com/openshift3/logging-deployer                       v3.5                a1d1ed79ef5d        7 days ago          885.9 MB
registry.access.redhat.com/openshift3/ose-haproxy-router                     v3.5                d3f9da00ddc3        7 days ago          744.3 MB
registry.access.redhat.com/openshift3/logging-auth-proxy                     v3.5                9cb817ba113b        7 days ago          215.3 MB
registry.access.redhat.com/openshift3/logging-curator                        v3.5                dbbf9432555f        7 days ago          211.3 MB
registry.access.redhat.com/openshift3/ose-sti-builder                        v3.5                0d120d98fc4a        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-docker-builder                     v3.5                f645a3b5fd83        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-recycler                           v3.5                31f3d2587aaa        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-deployer                           v3.5                14abad07b930        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-docker-registry                    v3.5                d71f844c95a9        7 days ago          791.3 MB
registry.access.redhat.com/openshift3/ose                                    v3.5                2116427b6df0        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-keepalived-ipfailover              v3.5                de878af82984        7 days ago          381.4 MB
registry.access.redhat.com/openshift3/ose-pod                                v3.5                be92a7c78132        7 days ago          205.2 MB
registry.access.redhat.com/jboss-eap-7/eap70-openshift                       latest              fa3b5c0231ae        12 days ago         1.042 GB
registry.access.redhat.com/jboss-webserver-3/webserver30-tomcat8-openshift   latest              1191bc03e0a0        12 days ago         687.9 MB
registry.access.redhat.com/rhscl/mysql-56-rhel7                              latest              02c6c6b90f8b        3 weeks ago         366.5 MB
registry.access.redhat.com/rhscl/mongodb-32-rhel7                            latest              def11b64d092        3 weeks ago         555.1 MB
registry.access.redhat.com/openshift3/jenkins-2-rhel7                        latest              2d3a35f67c4d        4 weeks ago         666.9 MB
registry.access.redhat.com/openshift3/jenkins-1-rhel7                        latest              9e95f7537703        4 weeks ago         661.7 MB
registry.access.redhat.com/rhscl/ruby-23-rhel7                               latest              11ab0a5df4b4        4 weeks ago         458.8 MB
registry.access.redhat.com/rhscl/python-35-rhel7                             latest              8934ad9b6a98        4 weeks ago         530.5 MB
registry.access.redhat.com/rhscl/php-56-rhel7                                latest              7064989edfe2        4 weeks ago         484.7 MB
registry.access.redhat.com/rhscl/redis-32-rhel7                              latest              16a06c303765        4 weeks ago         218.8 MB
registry.access.redhat.com/openshift3/nodejs-010-rhel7                       latest              226d0b1b7987        4 months ago        430.2 MB

确认推送成功后,可以删除 .tar.gz 文件
rm -rf *.tar.gz

4. 推送镜像到本地 Docker Registry在 Registry 机器上操作
cp /etc/sysconfig/docker /etc/sysconfig/docker.bak.$(date "+%Y%m%d%H%M%S");
sed  -i s/".*OPTIONS=.*"/"OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0\/16 --insecure-registry registry.example.com:5000'"/g /etc/sysconfig/docker;
sed -i 's/registry.access.redhat.com/registry.example.com:5000/g' /etc/sysconfig/docker;
systemctl restart docker

REDHAT_REG="registry.access.redhat.com";
PRIVATE_REG="registry.example.com:5000";
for i in $(docker images|grep $REDHAT_REG|awk '{print $1":"$2}') ; do docker tag  $i "$PRIVATE_REG$(echo $i|awk -F 'com' {'print $2'})" ; done;
for i in `docker images|grep $PRIVATE_REG|awk '{print $1":"$2}'` ; do  docker push $i; done;
for i in $(docker images|grep $REDHAT_REG|awk '{print $1":"$2}') ; do docker rmi $i ; done;

5. 修改核心镜像为具体版本

5.1 打具体版本 tag 
docker tag registry.example.com:5000/openshift3/ose:v3.5 registry.example.com:5000/openshift3/ose:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-deployer:v3.5 registry.example.com:5000/openshift3/ose-deployer:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-sti-builder:v3.5 registry.example.com:5000/openshift3/ose-sti-builder:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-docker-builder:v3.5 registry.example.com:5000/openshift3/ose-docker-builder:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-pod:v3.5 registry.example.com:5000/openshift3/ose-pod:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5 registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-docker-registry:v3.5 registry.example.com:5000/openshift3/ose-docker-registry:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-recycler:v3.5 registry.example.com:5000/openshift3/ose-recycler:v3.5.5.15
docker tag registry.example.com:5000/openshift3/registry-console:v3.5 registry.example.com:5000/openshift3/registry-console:v3.5.5.15

5.2 删除核心镜像 tag v3.5
docker rmi registry.example.com:5000/openshift3/ose:v3.5
docker rmi registry.example.com:5000/openshift3/ose-haproxy-router:v3.5
docker rmi registry.example.com:5000/openshift3/ose-deployer:v3.5
docker rmi registry.example.com:5000/openshift3/ose-sti-builder:v3.5
docker rmi registry.example.com:5000/openshift3/ose-docker-builder:v3.5
docker rmi registry.example.com:5000/openshift3/ose-pod:v3.5
docker rmi registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5
docker rmi registry.example.com:5000/openshift3/ose-docker-registry:v3.5
docker rmi registry.example.com:5000/openshift3/ose-recycler:v3.5
docker rmi registry.example.com:5000/openshift3/registry-console:v3.5

5.3 push 核心镜像 tag v3.5.5.15
docker push registry.example.com:5000/openshift3/ose:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-haproxy-router:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-deployer:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-sti-builder:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-docker-builder:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-pod:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-docker-registry:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-recycler:v3.5.5.15

5.4 push registry-console 镜像 tag 3.5
docker push registry.example.com:5000/openshift3/registry-console:3.5

注意,registry-console 的 tag 必须是 3.5。

6. 查看所有镜像
docker images | sort
输出如下:
输出如下:
registry.example.com:5000/jboss-amq-6/amq62-openshift                         latest              2906bbb3656a        4 weeks ago         594.6 MB
registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift   latest              8aec98ccf8d8        12 days ago         1.072 GB
registry.example.com:5000/jboss-eap-6/eap64-openshift                         latest              00c5e5d517c6        13 days ago         1.009 GB
registry.example.com:5000/jboss-eap-7/eap70-openshift                         latest              fa3b5c0231ae        13 days ago         1.042 GB
registry.example.com:5000/jboss-processserver-6/processserver63-openshift     latest              10f82e4f3451        12 days ago         1.072 GB
registry.example.com:5000/jboss-webserver-3/webserver30-tomcat7-openshift     latest              9c1aa53845a5        13 days ago         686 MB
registry.example.com:5000/jboss-webserver-3/webserver30-tomcat8-openshift     latest              1191bc03e0a0        13 days ago         687.9 MB
registry.example.com:5000/openshift3/jenkins-1-rhel7                          latest              9e95f7537703        4 weeks ago         661.7 MB
registry.example.com:5000/openshift3/jenkins-2-rhel7                          latest              2d3a35f67c4d        4 weeks ago         666.9 MB
registry.example.com:5000/openshift3/jenkins-slave-base-rhel7                 latest              1efc4d59bd10        4 weeks ago         491.4 MB
registry.example.com:5000/openshift3/jenkins-slave-maven-rhel7                latest              53109d44c8ec        4 weeks ago         577.3 MB
registry.example.com:5000/openshift3/logging-auth-proxy                       v3.5                9cb817ba113b        8 days ago          215.3 MB
registry.example.com:5000/openshift3/logging-curator                          v3.5                dbbf9432555f        8 days ago          211.3 MB
registry.example.com:5000/openshift3/logging-deployer                         v3.5                a1d1ed79ef5d        8 days ago          885.9 MB
registry.example.com:5000/openshift3/logging-elasticsearch                    v3.5                1dca1f8506c0        8 days ago          399.5 MB
registry.example.com:5000/openshift3/logging-fluentd                          v3.5                1421c119d952        8 days ago          232.8 MB
registry.example.com:5000/openshift3/logging-kibana                           v3.5                f9bfc4c46fb5        8 days ago          342.6 MB
registry.example.com:5000/openshift3/metrics-cassandra                        v3.5                581c9cad4aaf        8 days ago          539.5 MB
registry.example.com:5000/openshift3/metrics-deployer                         v3.5                c044fb3618c3        8 days ago          892.7 MB
registry.example.com:5000/openshift3/metrics-hawkular-metrics                 v3.5                94cd687068c2        8 days ago          1.269 GB
registry.example.com:5000/openshift3/metrics-heapster                         v3.5                cc6454e9d765        8 days ago          317.9 MB
registry.example.com:5000/openshift3/nodejs-010-rhel7                         latest              226d0b1b7987        4 months ago        430.2 MB
registry.example.com:5000/openshift3/ose-deployer                             v3.5.5.15           14abad07b930        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose-docker-builder                       v3.5.5.15           f645a3b5fd83        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose-docker-registry                      v3.5.5.15           d71f844c95a9        8 days ago          791.3 MB
registry.example.com:5000/openshift3/ose-haproxy-router                       v3.5.5.15           d3f9da00ddc3        8 days ago          744.3 MB
registry.example.com:5000/openshift3/ose-keepalived-ipfailover                v3.5.5.15           de878af82984        8 days ago          381.4 MB
registry.example.com:5000/openshift3/ose-pod                                  v3.5.5.15           be92a7c78132        8 days ago          205.2 MB
registry.example.com:5000/openshift3/ose-recycler                             v3.5.5.15           31f3d2587aaa        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose-sti-builder                          v3.5.5.15           0d120d98fc4a        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose                                      v3.5.5.15           2116427b6df0        8 days ago          725.3 MB
registry.example.com:5000/openshift3/registry-console                         3.5                 e53d505ae98e        8 days ago          476.2 MB
registry.example.com:5000/rhscl/mongodb-32-rhel7                              latest              def11b64d092        3 weeks ago         555.1 MB
registry.example.com:5000/rhscl/mysql-56-rhel7                                latest              02c6c6b90f8b        3 weeks ago         366.5 MB
registry.example.com:5000/rhscl/mysql-57-rhel7                                latest              9acccac03cc8        3 weeks ago         401.4 MB
registry.example.com:5000/rhscl/php-56-rhel7                                  latest              7064989edfe2        4 weeks ago         484.7 MB
registry.example.com:5000/rhscl/php-70-rhel7                                  latest              47fba8f64019        4 weeks ago         482.5 MB
registry.example.com:5000/rhscl/python-35-rhel7                               latest              8934ad9b6a98        4 weeks ago         530.5 MB
registry.example.com:5000/rhscl/redis-32-rhel7                                latest              16a06c303765        4 weeks ago         218.8 MB
registry.example.com:5000/rhscl/ruby-23-rhel7                                 latest              11ab0a5df4b4        4 weeks ago         458.8 MB
registry.example.com:5000/rhscl/s2i-base-rhel7                                latest              c8b1a95b13d0        4 weeks ago         383 MB

7. 验证能否成功下载镜像 (在所有机器上操作)
docker pull registry.example.com:5000/rhscl/php-56-rhel7
docker pull registry.example.com:5000/openshift3/registry-console:3.5
docker pull registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift
docker pull registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker pull registry.example.com:5000/openshift3/logging-auth-proxy:v3.5

docker rmi registry.example.com:5000/rhscl/php-56-rhel7
docker rmi registry.example.com:5000/openshift3/registry-console:3.5
docker rmi registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift
docker rmi registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker rmi registry.example.com:5000/openshift3/logging-auth-proxy:v3.5

OpenShift_079:离线安装 OCP 3.5 之 环境准备 之 GIT 服务配置

1. 创建仓库(在 Registry 上操作)
mkdir -p /opt/git/repo/cakephp.git;
cd /opt/git/repo/cakephp.git;
git init --bare;
git update-server-info;
mv hooks/post-update.sample hooks/post-update;

2. 拷贝 cakephp-ex-master.zip 到 Registry 上 (在 MAC 上操作)
scp  cakephp-ex-master.zip root@192.168.56.112:/opt

3. 复制并提交第一份代码(在 Registry 上操作)
cd /opt;
git clone file:///opt/git/repo/cakephp.git/;
unzip cakephp-ex-master.zip;
cp cakephp-ex-master/* cakephp -rf;
cp cakephp-ex-master/.sti cakephp -rf;
cp cakephp-ex-master/.htaccess cakephp -rf;
cp cakephp-ex-master/.gitignore cakephp -rf;
cd cakephp;
git add .;
git commit -m 'initial upload';
git push origin master;

4. 发布仓库到 httpd(在 Registry 上操作)
创建文件/etc/httpd/conf.d/git.conf
cat << EOF > /etc/httpd/conf.d/git.conf
Alias /git "/opt/git/repo"
<Directory "/opt/git/repo">
    AllowOverride None
  Dav On
    Options +Indexes +FollowSymLinks
Require all granted
</Directory>
<Directory "/opt/git/repo/cakephp.git/">
    Require all granted
</Directory>
<Location /git>
SetHandler None
</Location>
EOF

5. 重启 httpd 使配置生效(在 Registry 上操作)
systemctl restart httpd

6. 配置 iptables 规则(在 Registry 上操作)
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 80 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;

7.  测试 GIT 服务(在所有机器上操作)
cd /tmp;
git clone http://git.example.com/git/cakephp.git/;
rm -rf cakephp;

OpenShift_078:离线安装 OCP 3.5 之 环境准备 之 DNS 服务配置

1. 修改 /etc/hosts(在 Master 机器上操作)
echo "192.168.56.111     master.example.com" >> /etc/hosts;
echo "192.168.56.112     registry.example.com" >> /etc/hosts;
echo "192.168.56.112     git.example.com" >> /etc/hosts;
echo "192.168.56.112     yum.example.com" >> /etc/hosts;
echo "192.168.56.112     nfs.example.com" >> /etc/hosts;
echo "192.168.56.113     node1.example.com" >> /etc/hosts;
echo "192.168.56.114     node2.example.com" >> /etc/hosts;

2.  添加 dnsmasq 配置,添加 wildcard 域名指向(在 Master 机器上操作)
cat > /etc/dnsmasq.d/openshift-cluster.conf <<EOF
local=/example.com/
address=/.apps.example.com/192.168.56.113
EOF


说明:192.168.56.113 是 router 所在的 node 节点,这里是 Node1。

3.  启动 dnsmasq 服务(在 Master 机器上操作)
systemctl start dnsmasq;
systemctl enable dnsmasq;

4.  修改 iptables 规则(在 Master 机器上操作)
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;
systemctl enable iptables;

5. 配置 Master 域名解析(在 Master 机器上操作)
systemctl restart NetworkManager;
nmcli con mod $(nmcli  con show |grep -v docker|head -2|tail -1|awk '{print $1}') ipv4.dns $(hostname -i);
或者 nmcli con mod enp0s8 ipv4.dns 192.168.56.111
systemctl restart NetworkManager;

说明:该命令的结果是在网卡 enp0s8 的配置中增加一项:DNS1=192.168.56.111

6.  配置各节点域名解析(在其它机器(除 Master)上操作)
cat > /etc/dnsmasq.d/openshift-cluster-node.conf <<EOF
server=192.168.56.111
EOF
systemctl restart dnsmasq;
systemctl enable dnsmasq;

systemctl restart NetworkManager;
nmcli con mod $(nmcli  con show |grep -v docker|head -2|tail -1|awk '{print $1}') ipv4.dns $(hostname -i);
或者
nmcli con mod enp0s8 ipv4.dns 192.168.56.112 (在 Registry 机器上操作)
nmcli con mod enp0s8 ipv4.dns 192.168.56.113 (在 Node1 机器上操作)
nmcli con mod enp0s8 ipv4.dns 192.168.56.114 (在 Node2 机器上操作)
systemctl restart NetworkManager;

7. 测试 DNS 域名解析(在所有机器上操作)
ping master.example.com -c 3;
ping node1.example.com -c 3;
ping node2.example.com -c 3;
ping registry.example.com -c 3;
ping git.example.com -c 3;
ping yum.example.com -c 3;
ping apps.example.com -c 3;

8. 原理说明
(1)每台机器的网卡 enp0s8 的 DNS1 的配置都指向自己机器的 IP 地址
(2)每台机器(除 Master 以外)的 /etc/dnsmasq.d/openshift-cluster-node.conf 的配置都指向 Master 机器的 IP 地址
(3)Master 机器的 /etc/dnsmasq.d/openshift-cluster.conf 的配置都指向 router 所在机器的 IP 地址。

OpenShift_077:离线安装 OCP 3.5 之 环境准备 之 操作系统配置

1. 设置主机域名(在所有机器上分别操作)
hostnamectl set-hostname master.example.com(在 Master 机器上操作)
hostnamectl set-hostname registry.example.com(在 Registry 机器上操作)
hostnamectl set-hostname node1.example.com(在 Node1 机器上操作)
hostnamectl set-hostname node2.example.com(在 Node2 机器上操作)

2. 安装软件包(在所有机器上操作)
yum -y install wget git net-tools bind-utils iptables-services bridge-utils bash-completion vim lrzsz unzip;
yum -y update;
yum -y install atomic-openshift-utils;

3. 安装配置 Docker(在所有机器上操作)
yum -y install docker;
systemctl enable docker;
cp /etc/sysconfig/docker /etc/sysconfig/docker.bak.$(date "+%Y%m%d%H%M%S");
sed  -i s/".*OPTIONS=.*"/"OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0\/16 --insecure-registry registry.example.com:5000'"/g /etc/sysconfig/docker;
sed -i 's/registry.access.redhat.com/registry.example.com:5000/g' /etc/sysconfig/docker;
docker-storage-setup;
输出如下:
ERROR: There is not enough free space in volume group rhel to create data volume of size MIN_DATA_SIZE=2G.

确认 SELinux 为 Enforcing 状态
getenforce;

4. 重启(在所有机器上操作)
reboot;

OpenShift_076:离线安装 OCP 3.5 之 环境准备 之 配置 YUM 源

1. 机器部署说明
(1)Master
IP 地址:192.168.56.111
域名:master.example.com
功能:Master、DNS Server
(2)Registry
IP 地址: 192.168.56.112
域名:registry.example.comgit.example.comyum.example.comnfs.example.com
功能:GIT Server、YUM Server、DNS Server、本地 Docker Registry
(3)Node1
IP 地址: 192.168.56.113
域名:node1.example.com
功能:Node、Router、内部 Docker Registry
(4)Node2
IP 地址: 192.168.56.114
域名:node2.example.com
功能:Node

2.  操作系统
所有机器均安装 Red Hat Enterprise Linux 7.3(选择 Minimal 模式)

3. 网络
所有机器均使用静态 IP 地址。
以 registry.example.com 机器为例,其它机器以此类推:

vi /etc/sysconfig/network-scripts/ifcfg-enp0s8

TYPE=Ethernet
BOOTPROTO=none
DEFROUTE=yes
#PEERDNS=yes
#PEERROUTES=yes
IPV4_FAILURE_FATAL=no
#IPV6INIT=yes
#IPV6_AUTOCONF=yes
#IPV6_DEFROUTE=yes
#IPV6_PEERDNS=yes
#IPV6_PEERROUTES=yes
#IPV6_FAILURE_FATAL=no
#IPV6_ADDR_GEN_MODE=stable-privacy
NAME=enp0s8
UUID=625caf3e-78d3-4075-9256-8c5924bedda4
DEVICE=enp0s8
ONBOOT=yes
IPADDR=192.168.56.112
PREFIX=24
GATEWAY=192.168.56.1

注意,为避免冲突,每个机器的UUID必须保证唯一。
可以运行 uuidgen 生成新的 UUID。

4. 创建本地 YUM 源(在 Registry 上操作)
mkdir /opt/ose

4.1 将预先下载好的 OpenShift Repo 拷贝至 Registry (在 MAC 上操作)
scp -r rhel-7-server-extras-rpms root@192.168.56.112:/opt/ose/
scp -r rhel-7-server-ose-3.5-rpms root@192.168.56.112:/opt/ose/
scp -r rhel-7-server-rpms root@192.168.56.112:/opt/ose/
scp -r rhel-7-fast-datapath-rpms root@192.168.56.112:/opt/ose/

4.2. 安装 createrepo工具(在 Registry 上操作)
cd /opt/ose/rhel-7-server-rpms/Packages/
rpm -Uvh createrepo-0.9.9-26.el7.noarch.rpm deltarpm-3.6-3.el7.x86_64.rpm python-deltarpm-3.6-3.el7.x86_64.rpm

4.3. 更新 repodata(在 Registry 上操作)
createrepo --worker=5 /opt/ose;

4.4. 停止并禁用防火墙(在所有机器上操作)
systemctl stop firewalld;
systemctl disable firewalld;

4.5 安装配置 httpd (在 Registry 上操作)
cat << EOF > /etc/yum.repos.d/local.repo
[local]
name = local
baseurl = file:///opt/ose
gpgcheck = 0
enabled = 1
EOF

yum clean all;
yum -y install httpd;

创建 httpd 配置文件,以发布 YUM 源。
cat << EOF > /etc/httpd/conf.d/yum.conf
Alias /repo "/opt/ose"
<Directory "/opt/ose">
  Options +Indexes +FollowSymLinks
  Require all granted
</Directory>
<Location /repo>
  SetHandler None
</Location>
EOF

重启 httpd,使修改生效
systemctl enable httpd;
systemctl restart httpd;

4.6 测试所有节点的YUM 配置,确保能正确连接到内部YUM服务器。(在所有机器上操作)
cat << EOF > /etc/yum.repos.d/ose.repo
[OpenShift]
baseurl = http://192.168.56.112/repo/
gpgcheck = 0
enabled = 1
EOF

yum list | grep -i atomic-openshift;
输出如下:
Repository 'OpenShift' is missing name in configuration, using id
atomic-openshift.x86_64                3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-clients.x86_64        3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-clients-redistributable.x86_64
atomic-openshift-docker-excluder.noarch
atomic-openshift-dockerregistry.x86_64 3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-excluder.noarch       3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-master.x86_64         3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-node.x86_64           3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-pod.x86_64            3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-sdn-ovs.x86_64        3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-tests.x86_64          3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-utils.noarch          3.5.71-1.git.0.128c2db.el7  OpenShift
tuned-profiles-atomic-openshift-node.x86_64

2017年5月21日星期日

OpenShift_075:离线安装 OCP 3.5 之 下载安装介质

环境 OCP 3.5

OpenShift Container Platform 3.5 于 2017年4月18日正式发布。

1. 安装 RHEL 7.3 
所有操作以 root 用户进行。

2. 注册订阅账户
subscription-manager register --username=<user_name> --password=<password>

3. 查找包含 OpenShift 订阅的 Pool ID
subscription-manager list --available;
subscription-manager list --available --matches '*OpenShift*'

4. 绑定到你的 Pool ID
subscription-manager attach --pool=<YOUR_POOL_ID>;

5. 启用 OpenShift 需要的 Repo
禁止所有的 repo
subscription-manager repos --disable="*"

查看是否还有未被禁止的 repo
yum repolist

如果还有未被禁止的 repo,手工禁止它
yum-config-manager --disable <repo_id>
yum-config-manager --disable \*

仅启用 OCP 3.5 所需的 repo
subscription-manager repos \
    --enable="rhel-7-server-rpms" \
    --enable="rhel-7-server-extras-rpms" \
    --enable="rhel-7-server-ose-3.5-rpms" \
    --enable="rhel-7-fast-datapath-rpms"

注意,OCP 3.5 比 3.4 多了一个 repo:rhel-7-fast-datapath-rpms。

6. 安装 reposync 命令
yum install -y yum-utils;

7. 下载 OpenShift 需要的 Repo
reposync -p /opt/ose -n;

8. 下载红帽官方镜像
红帽官方镜像查询和下载地址:https://access.redhat.com/containers/

REGISTRY="registry.access.redhat.com";PTH="openshift3";VERSION="v3.5";

8.1 下载核心镜像
docker pull $REGISTRY/$PTH/ose:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-haproxy-router:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-deployer:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-sti-builder:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-docker-builder:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-pod:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-keepalived-ipfailover:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-docker-registry:$VERSION; \
docker pull $REGISTRY/$PTH/ose-recycler:$VERSION; \
docker pull $REGISTRY/$PTH/registry-console:$VERSION;

导出镜像为 tar.gz
docker save -o ose-images-core-$VERSION_`date +'%Y%m%d'`.tar.gz $REGISTRY/$PTH/ose:$VERSION $REGISTRY/$PTH/ose-haproxy-router:$VERSION $REGISTRY/$PTH/ose-deployer:$VERSION $REGISTRY/$PTH/ose-sti-builder:$VERSION $REGISTRY/$PTH/ose-docker-builder:$VERSION $REGISTRY/$PTH/ose-pod:$VERSION $REGISTRY/$PTH/ose-keepalived-ipfailover:$VERSION $REGISTRY/$PTH/ose-docker-registry:$VERSION $REGISTRY/$PTH/ose-recycler:$VERSION $REGISTRY/$PTH/registry-console:$VERSION

8.2 下载 logging 和 metrics 镜像
docker pull $REGISTRY/$PTH/logging-deployer:$VERSION; \
docker pull $REGISTRY/$PTH/logging-elasticsearch:$VERSION; \
docker pull $REGISTRY/$PTH/logging-kibana:$VERSION; \
docker pull $REGISTRY/$PTH/logging-fluentd:$VERSION; \
docker pull $REGISTRY/$PTH/logging-auth-proxy:$VERSION; \
docker pull $REGISTRY/$PTH/logging-curator:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-deployer:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-hawkular-metrics:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-cassandra:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-heapster:$VERSION;

导出镜像为 tar.gz
docker save -o ose-images-logging_metric-$VERSION_`date +'%Y%m%d'`.tar.gz $REGISTRY/$PTH/logging-deployer:$VERSION $REGISTRY/$PTH/logging-elasticsearch:$VERSION $REGISTRY/$PTH/logging-kibana:$VERSION $REGISTRY/$PTH/logging-fluentd:$VERSION $REGISTRY/$PTH/logging-auth-proxy:$VERSION $REGISTRY/$PTH/logging-curator:$VERSION $REGISTRY/$PTH/metrics-deployer:$VERSION $REGISTRY/$PTH/metrics-hawkular-metrics:$VERSION $REGISTRY/$PTH/metrics-cassandra:$VERSION $REGISTRY/$PTH/metrics-heapster:$VERSION

8.3 下载 apps 镜像
docker pull $REGISTRY/jboss-webserver-3/webserver30-tomcat7-openshift:latest; \
docker pull $REGISTRY/jboss-webserver-3/webserver30-tomcat8-openshift:latest; \
docker pull $REGISTRY/jboss-eap-6/eap64-openshift:latest; \
docker pull $REGISTRY/jboss-eap-7/eap70-openshift:latest; \

docker pull $REGISTRY/jboss-amq-6/amq62-openshift:latest; \
docker pull $REGISTRY/jboss-fuse-6/fis-java-openshift:latest; \
docker pull $REGISTRY/jboss-fuse-6/fis-karaf-openshift:latest; \
docker pull $REGISTRY/jboss-processserver-6/processserver63-openshift:latest; \
docker pull $REGISTRY/jboss-decisionserver-6/decisionserver63-openshift:latest; \

docker pull $REGISTRY/rhscl/mongodb-32-rhel7:latest; \
docker pull $REGISTRY/rhscl/mysql-56-rhel7:latest; \
docker pull $REGISTRY/rhscl/mysql-57-rhel7:latest; \
docker pull $REGISTRY/rhscl/php-56-rhel7:latest; \
docker pull $REGISTRY/rhscl/php-70-rhel7:latest; \
docker pull $REGISTRY/rhscl/python-35-rhel7:latest; \
docker pull $REGISTRY/rhscl/redis-32-rhel7:latest; \
docker pull $REGISTRY/rhscl/ruby-23-rhel7:latest; \
docker pull $REGISTRY/rhscl/s2i-base-rhel7:latest; \

docker pull $REGISTRY/$PTH/jenkins-1-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-2-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-slave-base-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-slave-maven-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-slave-nodejs-rhel7:latest; \
docker pull $REGISTRY/$PTH/nodejs-010-rhel7:latest;

导出 apps 镜像为 tar.gz
docker save -o ose-images-apps-latest_`date +'%Y%m%d'`.tar.gz $REGISTRY/jboss-webserver-3/webserver30-tomcat8-openshift:latest $REGISTRY/jboss-eap-7/eap70-openshift:latest $REGISTRY/rhscl/mongodb-32-rhel7:latest $REGISTRY/rhscl/mysql-56-rhel7:latest $REGISTRY/rhscl/php-56-rhel7:latest $REGISTRY/rhscl/python-35-rhel7:latest $REGISTRY/rhscl/redis-32-rhel7:latest $REGISTRY/rhscl/ruby-23-rhel7:latest $REGISTRY/$PTH/jenkins-1-rhel7:latest $REGISTRY/$PTH/jenkins-2-rhel7:latest $REGISTRY/$PTH/nodejs-010-rhel7:latest;

导出 extra apps 镜像为 tar.gz
docker save -o ose-images-apps-extra-latest_`date +'%Y%m%d'`.tar.gz $REGISTRY/jboss-webserver-3/webserver30-tomcat7-openshift:latest $REGISTRY/jboss-eap-6/eap64-openshift:latest $REGISTRY/jboss-amq-6/amq62-openshift:latest $REGISTRY/jboss-processserver-6/processserver63-openshift:latest $REGISTRY/jboss-decisionserver-6/decisionserver63-openshift:latest $REGISTRY/rhscl/mysql-57-rhel7:latest $REGISTRY/rhscl/php-70-rhel7:latest $REGISTRY/rhscl/s2i-base-rhel7:latest $REGISTRY/$PTH/jenkins-slave-base-rhel7:latest $REGISTRY/$PTH/jenkins-slave-maven-rhel7:latest;