OpenShift_082：离线安装 OCP 3.5 之安装后配置

1. 配置用户（在 Master 上操作）
安装 httpd-tools
yum -y install httpd-tools;
创建一个 admin 用户，并且赋予权限
htpasswd -cb /etc/origin/master/htpasswd admin redhat;
oadm policy add-cluster-role-to-user admin admin

2. 配置二级域名（在 Master 上操作）
cp /etc/origin/master/master-config.yaml /etc/origin/master/master-config.yaml.bak.$(date "+%Y%m%d%H%M%S");
sed -i 's/.*subdomain.*/  subdomain: apps.example.com/' /etc/origin/master/master-config.yaml;

3. 重启 OpenShift，使上述修改生效（在 Master 上操作）
systemctl restart atomic-openshift-{master,node};

4. 如果之前自动安装 Registry Console 失败，以下手动重新安装

4.1 之前自动安装 Registry Console 有错，执行以下命令清除
oc delete dc registry-console
oc delete svc registry-console
oc delete route registry-console
oc delete is registry-console
oc delete oauthclients cockpit-oauth-client

4.2 创建 registry-console route
oc create route passthrough --service registry-console --port registry-console -n default

4.3 部署 registry console 应用
oc new-app -n default --template=registry-console \
    -p IMAGE_PREFIX="registry.example.com:5000/openshift3/" \
    -p IMAGE_VERSION="3.5" \
    -p OPENSHIFT_OAUTH_PROVIDER_URL="https://master.example.com:8443" \
    -p REGISTRY_HOST=$(oc get route docker-registry -n default --template='{{ .spec.host }}') \
    -p COCKPIT_KUBE_URL=$(oc get route registry-console -n default --template='https://{{ .spec.host }}')

5. 添加 Image Stream（在 Master 上操作）

5.1 查看当前的 Image Stream 
oc get is -n openshift
输出如下：
NAME                                  DOCKER REPO                                                                    TAGS                      UPDATED
dotnet                                172.30.132.67:5000/openshift/dotnet                                            1.1,1.0                
fis-java-openshift                    registry.access.redhat.com/jboss-fuse-6/fis-java-openshift                                            
fis-karaf-openshift                   registry.access.redhat.com/jboss-fuse-6/fis-karaf-openshift                                            
jboss-amq-62                          registry.access.redhat.com/jboss-amq-6/amq62-openshift                                                
jboss-datagrid65-openshift            registry.access.redhat.com/jboss-datagrid-6/datagrid65-openshift                                      
jboss-datavirt63-openshift            registry.access.redhat.com/jboss-datavirt-6/datavirt63-openshift                                      
jboss-decisionserver62-openshift      registry.access.redhat.com/jboss-decisionserver-6/decisionserver62-openshift                          
jboss-decisionserver63-openshift      registry.access.redhat.com/jboss-decisionserver-6/decisionserver63-openshift                          
jboss-eap64-openshift                 registry.access.redhat.com/jboss-eap-6/eap64-openshift                                                
jboss-eap70-openshift                 registry.access.redhat.com/jboss-eap-7/eap70-openshift                                                
jboss-processserver63-openshift       registry.access.redhat.com/jboss-processserver-6/processserver63-openshift                            
jboss-webserver30-tomcat7-openshift   registry.access.redhat.com/jboss-webserver-3/webserver30-tomcat7-openshift                            
jboss-webserver30-tomcat8-openshift   registry.access.redhat.com/jboss-webserver-3/webserver30-tomcat8-openshift                            
jenkins                               172.30.132.67:5000/openshift/jenkins                                           2,1                    
mariadb                               172.30.132.67:5000/openshift/mariadb                                           10.1                    
mongodb                               172.30.132.67:5000/openshift/mongodb                                           3.2,2.6,2.4            
mysql                                 172.30.132.67:5000/openshift/mysql                                             5.7,5.6,5.5            
nodejs                                172.30.132.67:5000/openshift/nodejs                                            0.10,4                  
perl                                  172.30.132.67:5000/openshift/perl                                              5.24,5.20,5.16          
php                                   172.30.132.67:5000/openshift/php                                               7.0,5.6,5.5            
postgresql                            172.30.132.67:5000/openshift/postgresql                                        9.4,9.2,9.5            
python                                172.30.132.67:5000/openshift/python                                            3.5,3.4,3.3 + 1 more...
redhat-openjdk18-openshift            registry.access.redhat.com/redhat-openjdk-18/openjdk18-openshift                                      
redhat-sso70-openshift                registry.access.redhat.com/redhat-sso-7/sso70-openshift                                                
redis                                 172.30.132.67:5000/openshift/redis                                             3.2                    
ruby                                  172.30.132.67:5000/openshift/ruby                                              2.3,2.2,2.0            

5.2 删除原有的 Image Stream
for i in $(oc get is -n openshift --no-headers|awk '{print $1}') ; do oc delete is $i -n openshift; done;

5.3 创建指向本地 Docker Registry 的 Image Stream

REDHAT_REG="registry.access.redhat.com";
PRIVATE_REG="registry.example.com:5000";

sed s/"${REDHAT_REG}"/"${PRIVATE_REG}"/g  /usr/share/openshift/examples/image-streams/image-streams-rhel7.json |sed '/"creationTimestamp": null/a\\t,"annotations": {"openshift.io/image.insecureRepository": "true"}' |oc create -n openshift -f - ;

sed s/"${REDHAT_REG}"/"${PRIVATE_REG}"/g /usr/share/openshift/examples/xpaas-streams/jboss-image-streams.json |sed '/"creationTimestamp": null/a\\t,"annotations": {"openshift.io/image.insecureRepository": "true"}' |oc create -n openshift -f - ;

sed s/"${REDHAT_REG}"/"${PRIVATE_REG}"/g /usr/share/openshift/examples/image-streams/dotnet_imagestreams.json |sed '/"creationTimestamp": null/a\\t,"annotations": {"openshift.io/image.insecureRepository": "true"}' |oc create -n openshift -f - ;

5.4 再次查看 Image Stream
oc get is -n openshift
输出如下：
NAME                                  DOCKER REPO                                                                   TAGS                      UPDATED
dotnet                                172.30.132.67:5000/openshift/dotnet                                           1.0,1.1              
jboss-amq-62                          registry.example.com:5000/jboss-amq-6/amq62-openshift                                                
jboss-datagrid65-openshift            registry.example.com:5000/jboss-datagrid-6/datagrid65-openshift                                      
jboss-datavirt63-openshift            registry.example.com:5000/jboss-datavirt-6/datavirt63-openshift                                      
jboss-decisionserver62-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver62-openshift                          
jboss-decisionserver63-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift                          
jboss-eap64-openshift                 registry.example.com:5000/jboss-eap-6/eap64-openshift                                                
jboss-eap70-openshift                 registry.example.com:5000/jboss-eap-7/eap70-openshift                                                
jboss-processserver63-openshift       registry.example.com:5000/jboss-processserver-6/processserver63-openshift                            
jboss-webserver30-tomcat7-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat7-openshift                            
jboss-webserver30-tomcat8-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat8-openshift                            
jenkins                               172.30.132.67:5000/openshift/jenkins                                          1,2                    
mariadb                               172.30.132.67:5000/openshift/mariadb                                          10.1                    
mongodb                               172.30.132.67:5000/openshift/mongodb                                          3.2,2.6,2.4            
mysql                                 172.30.132.67:5000/openshift/mysql                                            5.5,5.7,5.6            
nodejs                                172.30.132.67:5000/openshift/nodejs                                           4,0.10                  
perl                                  172.30.132.67:5000/openshift/perl                                             5.24,5.20,5.16          
php                                   172.30.132.67:5000/openshift/php                                              5.6,5.5,7.0            
postgresql                            172.30.132.67:5000/openshift/postgresql                                       9.5,9.4,9.2            
python                                172.30.132.67:5000/openshift/python                                           3.4,3.3,2.7 + 1 more...
redhat-openjdk18-openshift            registry.example.com:5000/redhat-openjdk-18/openjdk18-openshift                                      
redhat-sso70-openshift                registry.example.com:5000/redhat-sso-7/sso70-openshift                                                
redis                                 172.30.132.67:5000/openshift/redis                                            3.2                    
ruby                                  172.30.132.67:5000/openshift/ruby                                             2.3,2.2,2.0                    

5.5 导入列表中所有 Image
可以看到有些 Image Stream 没有 Tag，需要导入 Image
for i in $(oc get is -n openshift --no-headers|awk '{print $1}'); do oc import-image $i --insecure --all -n openshift;done

注意，不是所有的 image 都能导入成功，这个跟导入的 Image 有关。
如果没有该 Image，是无法成功导入 Image Stream 的。
并且，一个软件有很多版本的 Image，导入了哪个 Image，哪个 Image Stream Tag 才能导入成功。
比如 mysql 这个 Image 我导入了 5.7 和 5.6 版本，在执行 oc import-image 时，会报告 5.7 和 5.6 成功导入，而 5.5 没有导入成功。
oc import-image mysql --insecure -n openshift
输出如下：
The import completed successfully.

Name: mysql
Namespace: openshift
Created: 3 minutes ago
Labels: <none>
Annotations: openshift.io/display-name=MySQL
openshift.io/image.dockerRepositoryCheck=2017-05-24T10:12:33Z
Docker Pull Spec: 172.30.132.67:5000/openshift/mysql
Unique Images: 2
Tags: 4

5.7 (latest)
  tagged from registry.example.com:5000/rhscl/mysql-57-rhel7:latest
    will use insecure HTTPS or HTTP connections

  Provides a MySQL 5.7 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/mysql-container/tree/master/5.7/README.md.
  Tags: mysql

  * registry.example.com:5000/rhscl/mysql-57-rhel7@sha256:bcec20eec2b4292ba3e0ba91e05f07b1dd8f04f4fda2eaf1e32e947f613975a4
      2 minutes ago

5.6
  tagged from registry.example.com:5000/rhscl/mysql-56-rhel7:latest
    will use insecure HTTPS or HTTP connections

  Provides a MySQL 5.6 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/mysql-container/tree/master/5.6/README.md.
  Tags: mysql

  * registry.example.com:5000/rhscl/mysql-56-rhel7@sha256:cd7981a6fab1f9ac08ee60348b9c79badbb32a858ead07198eb5a493cc13c741
      2 minutes ago

5.5
  tagged from registry.example.com:5000/openshift3/mysql-55-rhel7:latest

  Provides a MySQL 5.5 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/mysql-container/tree/master/5.5/README.md.
  Tags: hidden, mysql

  ! error: Import failed (NotFound): dockerimage "registry.example.com:5000/openshift3/mysql-55-rhel7:latest" not found
      2 minutes ago

5.6 再次查看 Image Stream
oc get is -n openshift
输出如下：
NAME                                  DOCKER REPO                                                                   TAGS                         UPDATED
dotnet                                172.30.132.67:5000/openshift/dotnet                                           1.1,1.0                    
jboss-amq-62                          registry.example.com:5000/jboss-amq-6/amq62-openshift                         latest                       18 seconds ago
jboss-datagrid65-openshift            registry.example.com:5000/jboss-datagrid-6/datagrid65-openshift               latest                    
jboss-datavirt63-openshift            registry.example.com:5000/jboss-datavirt-6/datavirt63-openshift               latest                    
jboss-decisionserver62-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver62-openshift   latest                    
jboss-decisionserver63-openshift      registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift   latest                       17 seconds ago
jboss-eap64-openshift                 registry.example.com:5000/jboss-eap-6/eap64-openshift                         latest                       17 seconds ago
jboss-eap70-openshift                 registry.example.com:5000/jboss-eap-7/eap70-openshift                         latest                       17 seconds ago
jboss-processserver63-openshift       registry.example.com:5000/jboss-processserver-6/processserver63-openshift     latest                       17 seconds ago
jboss-webserver30-tomcat7-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat7-openshift     latest                       16 seconds ago
jboss-webserver30-tomcat8-openshift   registry.example.com:5000/jboss-webserver-3/webserver30-tomcat8-openshift     latest                       16 seconds ago
jenkins                               172.30.132.67:5000/openshift/jenkins                                          latest,2,1                   16 seconds ago
mariadb                               172.30.132.67:5000/openshift/mariadb                                          10.1                      
mongodb                               172.30.132.67:5000/openshift/mongodb                                          latest,3.2,2.4 + 1 more...   16 seconds ago
mysql                                 172.30.132.67:5000/openshift/mysql                                            latest,5.7,5.6 + 1 more...   15 seconds ago
nodejs                                172.30.132.67:5000/openshift/nodejs                                           0.10,4                    
perl                                  172.30.132.67:5000/openshift/perl                                             5.24,5.20,5.16            
php                                   172.30.132.67:5000/openshift/php                                              latest,7.0,5.6 + 1 more...   15 seconds ago
postgresql                            172.30.132.67:5000/openshift/postgresql                                       9.4,9.2,9.5                
python                                172.30.132.67:5000/openshift/python                                           latest,3.5,3.4 + 2 more...   15 seconds ago
redhat-openjdk18-openshift            registry.example.com:5000/redhat-openjdk-18/openjdk18-openshift               latest                    
redhat-sso70-openshift                registry.example.com:5000/redhat-sso-7/sso70-openshift                        latest                    
redis                                 172.30.132.67:5000/openshift/redis                                            latest,3.2                   14 seconds ago
ruby                                  172.30.132.67:5000/openshift/ruby                                             latest,2.3,2.2 + 1 more...   14 seconds ago

可以看到有些镜像多了很多版本。

6. 访问 OpenShift 控制台
https://master.example.com:8443 admin/redhat

OpenShift_081：离线安装 OCP 3.5 之安装

1.  SSH 互信配置（在 Master 上操作）
ssh-keygen;

for i in  master.example.com node1.example.com node2.example.com;
do
ssh-copy-id $i;
done;

2. 安装（在 Master 上操作）
cat > /etc/ansible/hosts <<EOF
[OSEv3:children]
masters
nodes

[OSEv3:vars]
ansible_ssh_user=root
deployment_type=openshift-enterprise
openshift_release=v3.5

#oreg_url=registry.example.com:5000/openshift3/ose-${component}:${version}
openshift_docker_additional_registries=registry.example.com:5000
openshift_docker_insecure_registries=registry.example.com:5000
openshift_examples_modify_imagestreams=true
openshift_cockpit_deployer_prefix=registry.example.com:5000/openshift3/

openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}]

openshift_clock_enabled=true

openshift_hosted_router_selector="infra=true"
openshift_hosted_registry_selector="infra=true"

#openshift_master_cluster_method=native
#openshift_master_cluster_hostname=master.example.com
#openshift_master_cluster_public_hostname=master.example.com

#openshift_master_default_subdomain=apps.example.com

#openshift_node_kubelet_args={'pods-per-core': ['10'], 'max-pods':['25']}

[masters]
master.example.com

[nodes]
master.example.com
node1.example.com openshift_node_labels="{'infra': 'true'}"

node2.example.com

EOF

说明：
（1）注释 oreg_url=registry.example.com:5000/openshift3/ose-${component}:${version}
经过多次实验，发现这个参数会引起如下错误，因此将其注释：
TASK [openshift_master : Start and enable master] ******************************
FAILED - RETRYING: TASK: openshift_master : Start and enable master (1 retries left).
fatal: [master.example.com]: FAILED! => {
    "attempts": 1,
    "changed": false,
    "failed": true
}

MSG:

Unable to start service atomic-openshift-master: Job for atomic-openshift-master.service failed because the control process exited with error code. See "systemctl status atomic-openshift-master.service" and "journalctl -xe" for details

（2）增加 openshift_cockpit_deployer_prefix=registry.example.com:5000/openshift3/
这样就可以正确 pull 到 registry-console 镜像。如果不加，它会到 registry.access.redhat.com 去 pull 镜像，是无法成功的。

执行安装脚本
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml;

安装大概需要 11 分钟，安装成功，最后会输出：
PLAY RECAP *********************************************************************
localhost                  : ok=8    changed=0    unreachable=0    failed=0
master.example.com         : ok=524  changed=140  unreachable=0    failed=0
node1.example.com          : ok=217  changed=60   unreachable=0    failed=0
node2.example.com          : ok=217  changed=60   unreachable=0    failed=0

如果安装失败，查看错误，修改 hosts 脚本，然后运行以下命令清理环境，再重装
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml

卸载 atomic-openshift-utils（在各个机器上操作）
yum remove atomic-openshift-utils

3. 安装后检查（在 Master 上操作）
oc get node --show-labels
输出如下：
NAME                 STATUS                     AGE       LABELS
master.example.com   Ready,SchedulingDisabled   5m        beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master.example.com
node1.example.com    Ready                      5m        beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node1.example.com
node2.example.com    Ready                      5m        beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node2.example.com

oc get pod -o wide
输出如下：
NAME                       READY     STATUS    RESTARTS   AGE       IP               NODE
docker-registry-1-9v8rt    1/1       Running   0          6m        10.129.0.4       node1.example.com
registry-console-1-n8prf   1/1       Running   0          5m        10.130.0.3       node2.example.com
router-1-9vnv6             1/1       Running   0          6m        192.168.56.113   node1.example.com

这里 registry、router、registry-console 安装成功。
如果 registry-console 安装失败，也可以后面手工安装 registry-console。

至此，说明安装成功！

4. 检查所有机器的 /etc/sysconfig/docker 文件内容 （在 Master、Node1、Node2 上操作）
OpenShift 成功安装后，会重置 /etc/sysconfig/docker文件，为保证后面使用正常，需要把这些文件改成正确的内容。
经过确认，Master、Node1、Node2 上的这个文件都是正确的。
如果不放心，运行以下脚本：
cp /etc/sysconfig/docker /etc/sysconfig/docker.bak.$(date "+%Y%m%d%H%M%S");
sed  -i s/".*OPTIONS=.*"/"OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0\/16 --insecure-registry registry.example.com:5000'"/g /etc/sysconfig/docker;
sed -i 's/registry.access.redhat.com/registry.example.com:5000/g' /etc/sysconfig/docker;
systemctl restart docker

OpenShift_080：离线安装 OCP 3.5 之 环境准备 之 本地 Docker Registry 配置

1.  安装并配置 Docker Registry （在 Registry 上操作）
yum -y install docker-distribution;
systemctl enable docker-distribution;
systemctl start docker-distribution;

mkdir /opt/ose/images

2. 拷贝之前下载的镜像到 Registry（在 MAC 上操作）
scp ose-images-core-20170522.tar.gz root@192.168.56.112:/opt/ose/images/
scp ose-images-logging_metric-20170522.tar.gz root@192.168.56.112:/opt/ose/images/
scp ose-images-apps-latest_20170522.tar.gz root@192.168.56.112:/opt/ose/images/
scp ose-images-apps-extra-latest_20170522.tar.gz root@192.168.56.112:/opt/ose/images/

3. 加载所有镜像（在 Registry 上操作）
cd /opt/ose/images/
for i in `ls *.tar.gz` ; do docker load -i $i; done;

docker images;

输出如下：
REPOSITORY                                                                   TAG                 IMAGE ID            CREATED             SIZE
registry.access.redhat.com/openshift3/metrics-hawkular-metrics               v3.5                94cd687068c2        7 days ago          1.269 GB
registry.access.redhat.com/openshift3/metrics-heapster                       v3.5                cc6454e9d765        7 days ago          317.9 MB
registry.access.redhat.com/openshift3/registry-console                       v3.5                e53d505ae98e        7 days ago          476.2 MB
registry.access.redhat.com/openshift3/logging-kibana                         v3.5                f9bfc4c46fb5        7 days ago          342.6 MB
registry.access.redhat.com/openshift3/metrics-cassandra                      v3.5                581c9cad4aaf        7 days ago          539.5 MB
registry.access.redhat.com/openshift3/logging-fluentd                        v3.5                1421c119d952        7 days ago          232.8 MB
registry.access.redhat.com/openshift3/logging-elasticsearch                  v3.5                1dca1f8506c0        7 days ago          399.5 MB
registry.access.redhat.com/openshift3/metrics-deployer                       v3.5                c044fb3618c3        7 days ago          892.7 MB
registry.access.redhat.com/openshift3/logging-deployer                       v3.5                a1d1ed79ef5d        7 days ago          885.9 MB
registry.access.redhat.com/openshift3/ose-haproxy-router                     v3.5                d3f9da00ddc3        7 days ago          744.3 MB
registry.access.redhat.com/openshift3/logging-auth-proxy                     v3.5                9cb817ba113b        7 days ago          215.3 MB
registry.access.redhat.com/openshift3/logging-curator                        v3.5                dbbf9432555f        7 days ago          211.3 MB
registry.access.redhat.com/openshift3/ose-sti-builder                        v3.5                0d120d98fc4a        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-docker-builder                     v3.5                f645a3b5fd83        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-recycler                           v3.5                31f3d2587aaa        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-deployer                           v3.5                14abad07b930        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-docker-registry                    v3.5                d71f844c95a9        7 days ago          791.3 MB
registry.access.redhat.com/openshift3/ose                                    v3.5                2116427b6df0        7 days ago          725.3 MB
registry.access.redhat.com/openshift3/ose-keepalived-ipfailover              v3.5                de878af82984        7 days ago          381.4 MB
registry.access.redhat.com/openshift3/ose-pod                                v3.5                be92a7c78132        7 days ago          205.2 MB
registry.access.redhat.com/jboss-eap-7/eap70-openshift                       latest              fa3b5c0231ae        12 days ago         1.042 GB
registry.access.redhat.com/jboss-webserver-3/webserver30-tomcat8-openshift   latest              1191bc03e0a0        12 days ago         687.9 MB
registry.access.redhat.com/rhscl/mysql-56-rhel7                              latest              02c6c6b90f8b        3 weeks ago         366.5 MB
registry.access.redhat.com/rhscl/mongodb-32-rhel7                            latest              def11b64d092        3 weeks ago         555.1 MB
registry.access.redhat.com/openshift3/jenkins-2-rhel7                        latest              2d3a35f67c4d        4 weeks ago         666.9 MB
registry.access.redhat.com/openshift3/jenkins-1-rhel7                        latest              9e95f7537703        4 weeks ago         661.7 MB
registry.access.redhat.com/rhscl/ruby-23-rhel7                               latest              11ab0a5df4b4        4 weeks ago         458.8 MB
registry.access.redhat.com/rhscl/python-35-rhel7                             latest              8934ad9b6a98        4 weeks ago         530.5 MB
registry.access.redhat.com/rhscl/php-56-rhel7                                latest              7064989edfe2        4 weeks ago         484.7 MB
registry.access.redhat.com/rhscl/redis-32-rhel7                              latest              16a06c303765        4 weeks ago         218.8 MB
registry.access.redhat.com/openshift3/nodejs-010-rhel7                       latest              226d0b1b7987        4 months ago        430.2 MB

确认推送成功后，可以删除 .tar.gz 文件
rm -rf *.tar.gz

4. 推送镜像到本地 Docker Registry（在 Registry 机器上操作）
cp /etc/sysconfig/docker /etc/sysconfig/docker.bak.$(date "+%Y%m%d%H%M%S");
sed  -i s/".*OPTIONS=.*"/"OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0\/16 --insecure-registry registry.example.com:5000'"/g /etc/sysconfig/docker;
sed -i 's/registry.access.redhat.com/registry.example.com:5000/g' /etc/sysconfig/docker;
systemctl restart docker

REDHAT_REG="registry.access.redhat.com";
PRIVATE_REG="registry.example.com:5000";
for i in $(docker images|grep $REDHAT_REG|awk '{print $1":"$2}') ; do docker tag  $i "$PRIVATE_REG$(echo $i|awk -F 'com' {'print $2'})" ; done;
for i in `docker images|grep $PRIVATE_REG|awk '{print $1":"$2}'` ; do  docker push $i; done;
for i in $(docker images|grep $REDHAT_REG|awk '{print $1":"$2}') ; do docker rmi $i ; done;

5. 修改核心镜像为具体版本

5.1 打具体版本 tag 
docker tag registry.example.com:5000/openshift3/ose:v3.5 registry.example.com:5000/openshift3/ose:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-deployer:v3.5 registry.example.com:5000/openshift3/ose-deployer:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-sti-builder:v3.5 registry.example.com:5000/openshift3/ose-sti-builder:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-docker-builder:v3.5 registry.example.com:5000/openshift3/ose-docker-builder:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-pod:v3.5 registry.example.com:5000/openshift3/ose-pod:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5 registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-docker-registry:v3.5 registry.example.com:5000/openshift3/ose-docker-registry:v3.5.5.15
docker tag registry.example.com:5000/openshift3/ose-recycler:v3.5 registry.example.com:5000/openshift3/ose-recycler:v3.5.5.15
docker tag registry.example.com:5000/openshift3/registry-console:v3.5 registry.example.com:5000/openshift3/registry-console:v3.5.5.15

5.2 删除核心镜像 tag v3.5
docker rmi registry.example.com:5000/openshift3/ose:v3.5
docker rmi registry.example.com:5000/openshift3/ose-haproxy-router:v3.5
docker rmi registry.example.com:5000/openshift3/ose-deployer:v3.5
docker rmi registry.example.com:5000/openshift3/ose-sti-builder:v3.5
docker rmi registry.example.com:5000/openshift3/ose-docker-builder:v3.5
docker rmi registry.example.com:5000/openshift3/ose-pod:v3.5
docker rmi registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5
docker rmi registry.example.com:5000/openshift3/ose-docker-registry:v3.5
docker rmi registry.example.com:5000/openshift3/ose-recycler:v3.5
docker rmi registry.example.com:5000/openshift3/registry-console:v3.5

5.3 push 核心镜像 tag v3.5.5.15

docker push registry.example.com:5000/openshift3/ose:v3.5.5.15

docker push registry.example.com:5000/openshift3/ose-haproxy-router:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-deployer:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-sti-builder:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-docker-builder:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-pod:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-docker-registry:v3.5.5.15
docker push registry.example.com:5000/openshift3/ose-recycler:v3.5.5.15

5.4 push registry-console 镜像 tag 3.5
docker push registry.example.com:5000/openshift3/registry-console:3.5

注意，registry-console 的 tag 必须是 3.5。

6. 查看所有镜像
docker images | sort
输出如下：
输出如下：
registry.example.com:5000/jboss-amq-6/amq62-openshift                         latest              2906bbb3656a        4 weeks ago         594.6 MB
registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift   latest              8aec98ccf8d8        12 days ago         1.072 GB
registry.example.com:5000/jboss-eap-6/eap64-openshift                         latest              00c5e5d517c6        13 days ago         1.009 GB
registry.example.com:5000/jboss-eap-7/eap70-openshift                         latest              fa3b5c0231ae        13 days ago         1.042 GB
registry.example.com:5000/jboss-processserver-6/processserver63-openshift     latest              10f82e4f3451        12 days ago         1.072 GB
registry.example.com:5000/jboss-webserver-3/webserver30-tomcat7-openshift     latest              9c1aa53845a5        13 days ago         686 MB
registry.example.com:5000/jboss-webserver-3/webserver30-tomcat8-openshift     latest              1191bc03e0a0        13 days ago         687.9 MB
registry.example.com:5000/openshift3/jenkins-1-rhel7                          latest              9e95f7537703        4 weeks ago         661.7 MB
registry.example.com:5000/openshift3/jenkins-2-rhel7                          latest              2d3a35f67c4d        4 weeks ago         666.9 MB
registry.example.com:5000/openshift3/jenkins-slave-base-rhel7                 latest              1efc4d59bd10        4 weeks ago         491.4 MB
registry.example.com:5000/openshift3/jenkins-slave-maven-rhel7                latest              53109d44c8ec        4 weeks ago         577.3 MB
registry.example.com:5000/openshift3/logging-auth-proxy                       v3.5                9cb817ba113b        8 days ago          215.3 MB
registry.example.com:5000/openshift3/logging-curator                          v3.5                dbbf9432555f        8 days ago          211.3 MB
registry.example.com:5000/openshift3/logging-deployer                         v3.5                a1d1ed79ef5d        8 days ago          885.9 MB
registry.example.com:5000/openshift3/logging-elasticsearch                    v3.5                1dca1f8506c0        8 days ago          399.5 MB
registry.example.com:5000/openshift3/logging-fluentd                          v3.5                1421c119d952        8 days ago          232.8 MB
registry.example.com:5000/openshift3/logging-kibana                           v3.5                f9bfc4c46fb5        8 days ago          342.6 MB
registry.example.com:5000/openshift3/metrics-cassandra                        v3.5                581c9cad4aaf        8 days ago          539.5 MB
registry.example.com:5000/openshift3/metrics-deployer                         v3.5                c044fb3618c3        8 days ago          892.7 MB
registry.example.com:5000/openshift3/metrics-hawkular-metrics                 v3.5                94cd687068c2        8 days ago          1.269 GB
registry.example.com:5000/openshift3/metrics-heapster                         v3.5                cc6454e9d765        8 days ago          317.9 MB
registry.example.com:5000/openshift3/nodejs-010-rhel7                         latest              226d0b1b7987        4 months ago        430.2 MB
registry.example.com:5000/openshift3/ose-deployer                             v3.5.5.15           14abad07b930        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose-docker-builder                       v3.5.5.15           f645a3b5fd83        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose-docker-registry                      v3.5.5.15           d71f844c95a9        8 days ago          791.3 MB
registry.example.com:5000/openshift3/ose-haproxy-router                       v3.5.5.15           d3f9da00ddc3        8 days ago          744.3 MB
registry.example.com:5000/openshift3/ose-keepalived-ipfailover                v3.5.5.15           de878af82984        8 days ago          381.4 MB
registry.example.com:5000/openshift3/ose-pod                                  v3.5.5.15           be92a7c78132        8 days ago          205.2 MB
registry.example.com:5000/openshift3/ose-recycler                             v3.5.5.15           31f3d2587aaa        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose-sti-builder                          v3.5.5.15           0d120d98fc4a        8 days ago          725.3 MB
registry.example.com:5000/openshift3/ose                                      v3.5.5.15           2116427b6df0        8 days ago          725.3 MB
registry.example.com:5000/openshift3/registry-console                         3.5                 e53d505ae98e        8 days ago          476.2 MB
registry.example.com:5000/rhscl/mongodb-32-rhel7                              latest              def11b64d092        3 weeks ago         555.1 MB
registry.example.com:5000/rhscl/mysql-56-rhel7                                latest              02c6c6b90f8b        3 weeks ago         366.5 MB
registry.example.com:5000/rhscl/mysql-57-rhel7                                latest              9acccac03cc8        3 weeks ago         401.4 MB
registry.example.com:5000/rhscl/php-56-rhel7                                  latest              7064989edfe2        4 weeks ago         484.7 MB
registry.example.com:5000/rhscl/php-70-rhel7                                  latest              47fba8f64019        4 weeks ago         482.5 MB
registry.example.com:5000/rhscl/python-35-rhel7                               latest              8934ad9b6a98        4 weeks ago         530.5 MB
registry.example.com:5000/rhscl/redis-32-rhel7                                latest              16a06c303765        4 weeks ago         218.8 MB
registry.example.com:5000/rhscl/ruby-23-rhel7                                 latest              11ab0a5df4b4        4 weeks ago         458.8 MB
registry.example.com:5000/rhscl/s2i-base-rhel7                                latest              c8b1a95b13d0        4 weeks ago         383 MB

7. 验证能否成功下载镜像 （在所有机器上操作）
docker pull registry.example.com:5000/rhscl/php-56-rhel7
docker pull registry.example.com:5000/openshift3/registry-console:3.5
docker pull registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift
docker pull registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker pull registry.example.com:5000/openshift3/logging-auth-proxy:v3.5

docker rmi registry.example.com:5000/rhscl/php-56-rhel7
docker rmi registry.example.com:5000/openshift3/registry-console:3.5
docker rmi registry.example.com:5000/jboss-decisionserver-6/decisionserver63-openshift
docker rmi registry.example.com:5000/openshift3/ose-keepalived-ipfailover:v3.5.5.15
docker rmi registry.example.com:5000/openshift3/logging-auth-proxy:v3.5

OpenShift_079：离线安装 OCP 3.5 之 环境准备 之 GIT 服务配置

1. 创建仓库（在 Registry 上操作）
mkdir -p /opt/git/repo/cakephp.git;
cd /opt/git/repo/cakephp.git;
git init --bare;
git update-server-info;
mv hooks/post-update.sample hooks/post-update;

2. 拷贝 cakephp-ex-master.zip 到 Registry 上 （在 MAC 上操作）
scp  cakephp-ex-master.zip root@192.168.56.112:/opt

3. 复制并提交第一份代码（在 Registry 上操作）
cd /opt;
git clone file:///opt/git/repo/cakephp.git/;
unzip cakephp-ex-master.zip;
cp cakephp-ex-master/* cakephp -rf;
cp cakephp-ex-master/.sti cakephp -rf;
cp cakephp-ex-master/.htaccess cakephp -rf;
cp cakephp-ex-master/.gitignore cakephp -rf;
cd cakephp;
git add .;
git commit -m 'initial upload';
git push origin master;

4. 发布仓库到 httpd（在 Registry 上操作）
创建文件/etc/httpd/conf.d/git.conf
cat << EOF > /etc/httpd/conf.d/git.conf
Alias /git "/opt/git/repo"
<Directory "/opt/git/repo">
    AllowOverride None
  Dav On
    Options +Indexes +FollowSymLinks
Require all granted
</Directory>
<Directory "/opt/git/repo/cakephp.git/">
    Require all granted
</Directory>
<Location /git>
SetHandler None
</Location>
EOF

5. 重启 httpd 使配置生效（在 Registry 上操作）
systemctl restart httpd

6. 配置 iptables 规则（在 Registry 上操作）
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 80 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;

7.  测试 GIT 服务（在所有机器上操作）
cd /tmp;
git clone http://git.example.com/git/cakephp.git/;
rm -rf cakephp;

OpenShift_078：离线安装 OCP 3.5 之 环境准备 之 DNS 服务配置

1. 修改 /etc/hosts（在 Master 机器上操作）
echo "192.168.56.111     master.example.com" >> /etc/hosts;
echo "192.168.56.112     registry.example.com" >> /etc/hosts;
echo "192.168.56.112     git.example.com" >> /etc/hosts;
echo "192.168.56.112     yum.example.com" >> /etc/hosts;
echo "192.168.56.112     nfs.example.com" >> /etc/hosts;
echo "192.168.56.113     node1.example.com" >> /etc/hosts;
echo "192.168.56.114     node2.example.com" >> /etc/hosts;

2.  添加 dnsmasq 配置，添加 wildcard 域名指向（在 Master 机器上操作）
cat > /etc/dnsmasq.d/openshift-cluster.conf <<EOF
local=/example.com/
address=/.apps.example.com/192.168.56.113
EOF

说明：192.168.56.113 是 router 所在的 node 节点，这里是 Node1。

3.  启动 dnsmasq 服务（在 Master 机器上操作）
systemctl start dnsmasq;
systemctl enable dnsmasq;

4.  修改 iptables 规则（在 Master 机器上操作）
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;
systemctl enable iptables;

5. 配置 Master 域名解析（在 Master 机器上操作）
systemctl restart NetworkManager;
nmcli con mod $(nmcli  con show |grep -v docker|head -2|tail -1|awk '{print $1}') ipv4.dns $(hostname -i);
或者 nmcli con mod enp0s8 ipv4.dns 192.168.56.111
systemctl restart NetworkManager;

说明：该命令的结果是在网卡 enp0s8 的配置中增加一项：DNS1=192.168.56.111。

6.  配置各节点域名解析（在其它机器（除 Master）上操作）
cat > /etc/dnsmasq.d/openshift-cluster-node.conf <<EOF
server=192.168.56.111
EOF
systemctl restart dnsmasq;
systemctl enable dnsmasq;

systemctl restart NetworkManager;
nmcli con mod $(nmcli  con show |grep -v docker|head -2|tail -1|awk '{print $1}') ipv4.dns $(hostname -i);
或者
nmcli con mod enp0s8 ipv4.dns 192.168.56.112 （在 Registry 机器上操作）
nmcli con mod enp0s8 ipv4.dns 192.168.56.113 （在 Node1 机器上操作）
nmcli con mod enp0s8 ipv4.dns 192.168.56.114 （在 Node2 机器上操作）
systemctl restart NetworkManager;

7. 测试 DNS 域名解析（在所有机器上操作）
ping master.example.com -c 3;
ping node1.example.com -c 3;
ping node2.example.com -c 3;
ping registry.example.com -c 3;
ping git.example.com -c 3;
ping yum.example.com -c 3;
ping apps.example.com -c 3;

8. 原理说明
（1）每台机器的网卡 enp0s8 的 DNS1 的配置都指向自己机器的 IP 地址。
（2）每台机器（除 Master 以外）的 /etc/dnsmasq.d/openshift-cluster-node.conf 的配置都指向 Master 机器的 IP 地址。
（3）Master 机器的 /etc/dnsmasq.d/openshift-cluster.conf 的配置都指向 router 所在机器的 IP 地址。

OpenShift_077：离线安装 OCP 3.5 之 环境准备 之 操作系统配置

1. 设置主机域名（在所有机器上分别操作）
hostnamectl set-hostname master.example.com（在 Master 机器上操作）
hostnamectl set-hostname registry.example.com（在 Registry 机器上操作）
hostnamectl set-hostname node1.example.com（在 Node1 机器上操作）
hostnamectl set-hostname node2.example.com（在 Node2 机器上操作）

2. 安装软件包（在所有机器上操作）
yum -y install wget git net-tools bind-utils iptables-services bridge-utils bash-completion vim lrzsz unzip;
yum -y update;
yum -y install atomic-openshift-utils;

3. 安装配置 Docker（在所有机器上操作）
yum -y install docker;
systemctl enable docker;
cp /etc/sysconfig/docker /etc/sysconfig/docker.bak.$(date "+%Y%m%d%H%M%S");
sed  -i s/".*OPTIONS=.*"/"OPTIONS='--selinux-enabled --insecure-registry 172.30.0.0\/16 --insecure-registry registry.example.com:5000'"/g /etc/sysconfig/docker;
sed -i 's/registry.access.redhat.com/registry.example.com:5000/g' /etc/sysconfig/docker;
docker-storage-setup;
输出如下：
ERROR: There is not enough free space in volume group rhel to create data volume of size MIN_DATA_SIZE=2G.

确认 SELinux 为 Enforcing 状态
getenforce;

4. 重启（在所有机器上操作）
reboot;

OpenShift_076：离线安装 OCP 3.5 之 环境准备 之 配置 YUM 源

1. 机器部署说明
（1）Master
IP 地址：192.168.56.111
域名：master.example.com
功能：Master、DNS Server
（2）Registry
IP 地址： 192.168.56.112
域名：registry.example.com、git.example.com、yum.example.com、nfs.example.com
功能：GIT Server、YUM Server、DNS Server、本地 Docker Registry
（3）Node1
IP 地址： 192.168.56.113
域名：node1.example.com
功能：Node、Router、内部 Docker Registry
（4）Node2
IP 地址： 192.168.56.114
域名：node2.example.com
功能：Node

2.  操作系统
所有机器均安装 Red Hat Enterprise Linux 7.3（选择 Minimal 模式）

3. 网络
所有机器均使用静态 IP 地址。
以 registry.example.com 机器为例，其它机器以此类推：

vi /etc/sysconfig/network-scripts/ifcfg-enp0s8

TYPE=Ethernet
BOOTPROTO=none
DEFROUTE=yes
#PEERDNS=yes
#PEERROUTES=yes
IPV4_FAILURE_FATAL=no
#IPV6INIT=yes
#IPV6_AUTOCONF=yes
#IPV6_DEFROUTE=yes
#IPV6_PEERDNS=yes
#IPV6_PEERROUTES=yes
#IPV6_FAILURE_FATAL=no
#IPV6_ADDR_GEN_MODE=stable-privacy
NAME=enp0s8
UUID=625caf3e-78d3-4075-9256-8c5924bedda4
DEVICE=enp0s8
ONBOOT=yes
IPADDR=192.168.56.112
PREFIX=24
GATEWAY=192.168.56.1

注意，为避免冲突，每个机器的UUID必须保证唯一。
可以运行 uuidgen 生成新的 UUID。

4. 创建本地 YUM 源（在 Registry 上操作）
mkdir /opt/ose

4.1 将预先下载好的 OpenShift Repo 拷贝至 Registry （在 MAC 上操作）
scp -r rhel-7-server-extras-rpms root@192.168.56.112:/opt/ose/
scp -r rhel-7-server-ose-3.5-rpms root@192.168.56.112:/opt/ose/
scp -r rhel-7-server-rpms root@192.168.56.112:/opt/ose/
scp -r rhel-7-fast-datapath-rpms root@192.168.56.112:/opt/ose/

4.2. 安装 createrepo工具（在 Registry 上操作）
cd /opt/ose/rhel-7-server-rpms/Packages/
rpm -Uvh createrepo-0.9.9-26.el7.noarch.rpm deltarpm-3.6-3.el7.x86_64.rpm python-deltarpm-3.6-3.el7.x86_64.rpm

4.3. 更新 repodata（在 Registry 上操作）
createrepo --worker=5 /opt/ose;

4.4. 停止并禁用防火墙（在所有机器上操作）
systemctl stop firewalld;
systemctl disable firewalld;

4.5 安装配置 httpd （在 Registry 上操作）
cat << EOF > /etc/yum.repos.d/local.repo
[local]
name = local
baseurl = file:///opt/ose
gpgcheck = 0
enabled = 1
EOF

yum clean all;
yum -y install httpd;

创建 httpd 配置文件，以发布 YUM 源。
cat << EOF > /etc/httpd/conf.d/yum.conf
Alias /repo "/opt/ose"
<Directory "/opt/ose">
  Options +Indexes +FollowSymLinks
  Require all granted
</Directory>
<Location /repo>
  SetHandler None
</Location>
EOF

重启 httpd，使修改生效
systemctl enable httpd;
systemctl restart httpd;

4.6 测试所有节点的YUM 配置，确保能正确连接到内部YUM服务器。（在所有机器上操作）
cat << EOF > /etc/yum.repos.d/ose.repo
[OpenShift]
baseurl = http://192.168.56.112/repo/
gpgcheck = 0
enabled = 1
EOF

yum list | grep -i atomic-openshift;
输出如下：
Repository 'OpenShift' is missing name in configuration, using id
atomic-openshift.x86_64                3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-clients.x86_64        3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-clients-redistributable.x86_64
atomic-openshift-docker-excluder.noarch
atomic-openshift-dockerregistry.x86_64 3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-excluder.noarch       3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-master.x86_64         3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-node.x86_64           3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-pod.x86_64            3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-sdn-ovs.x86_64        3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-tests.x86_64          3.5.5.15-1.git.0.4b5f317.el7
atomic-openshift-utils.noarch          3.5.71-1.git.0.128c2db.el7  OpenShift
tuned-profiles-atomic-openshift-node.x86_64

OpenShift_075：离线安装 OCP 3.5 之 下载安装介质

环境 OCP 3.5

OpenShift Container Platform 3.5 于 2017年4月18日正式发布。

1. 安装 RHEL 7.3 
所有操作以 root 用户进行。

2. 注册订阅账户
subscription-manager register --username=<user_name> --password=<password>

3. 查找包含 OpenShift 订阅的 Pool ID
subscription-manager list --available;
subscription-manager list --available --matches '*OpenShift*'

4. 绑定到你的 Pool ID
subscription-manager attach --pool=<YOUR_POOL_ID>;

5. 启用 OpenShift 需要的 Repo
禁止所有的 repo
subscription-manager repos --disable="*"

查看是否还有未被禁止的 repo
yum repolist

如果还有未被禁止的 repo，手工禁止它
yum-config-manager --disable <repo_id>
yum-config-manager --disable \*

仅启用 OCP 3.5 所需的 repo
subscription-manager repos \
    --enable="rhel-7-server-rpms" \
    --enable="rhel-7-server-extras-rpms" \
    --enable="rhel-7-server-ose-3.5-rpms" \
    --enable="rhel-7-fast-datapath-rpms"

注意，OCP 3.5 比 3.4 多了一个 repo：rhel-7-fast-datapath-rpms。

6. 安装 reposync 命令
yum install -y yum-utils;

7. 下载 OpenShift 需要的 Repo
reposync -p /opt/ose -n;

8. 下载红帽官方镜像
红帽官方镜像查询和下载地址：https://access.redhat.com/containers/

REGISTRY="registry.access.redhat.com";PTH="openshift3";VERSION="v3.5";

8.1 下载核心镜像
docker pull $REGISTRY/$PTH/ose:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-haproxy-router:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-deployer:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-sti-builder:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-docker-builder:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-pod:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-keepalived-ipfailover:$VERSION ; \
docker pull $REGISTRY/$PTH/ose-docker-registry:$VERSION; \
docker pull $REGISTRY/$PTH/ose-recycler:$VERSION; \
docker pull $REGISTRY/$PTH/registry-console:$VERSION;

导出镜像为 tar.gz
docker save -o ose-images-core-$VERSION_`date +'%Y%m%d'`.tar.gz $REGISTRY/$PTH/ose:$VERSION $REGISTRY/$PTH/ose-haproxy-router:$VERSION $REGISTRY/$PTH/ose-deployer:$VERSION $REGISTRY/$PTH/ose-sti-builder:$VERSION $REGISTRY/$PTH/ose-docker-builder:$VERSION $REGISTRY/$PTH/ose-pod:$VERSION $REGISTRY/$PTH/ose-keepalived-ipfailover:$VERSION $REGISTRY/$PTH/ose-docker-registry:$VERSION $REGISTRY/$PTH/ose-recycler:$VERSION $REGISTRY/$PTH/registry-console:$VERSION

8.2 下载 logging 和 metrics 镜像
docker pull $REGISTRY/$PTH/logging-deployer:$VERSION; \
docker pull $REGISTRY/$PTH/logging-elasticsearch:$VERSION; \
docker pull $REGISTRY/$PTH/logging-kibana:$VERSION; \
docker pull $REGISTRY/$PTH/logging-fluentd:$VERSION; \
docker pull $REGISTRY/$PTH/logging-auth-proxy:$VERSION; \
docker pull $REGISTRY/$PTH/logging-curator:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-deployer:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-hawkular-metrics:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-cassandra:$VERSION; \
docker pull $REGISTRY/$PTH/metrics-heapster:$VERSION;

导出镜像为 tar.gz
docker save -o ose-images-logging_metric-$VERSION_`date +'%Y%m%d'`.tar.gz $REGISTRY/$PTH/logging-deployer:$VERSION $REGISTRY/$PTH/logging-elasticsearch:$VERSION $REGISTRY/$PTH/logging-kibana:$VERSION $REGISTRY/$PTH/logging-fluentd:$VERSION $REGISTRY/$PTH/logging-auth-proxy:$VERSION $REGISTRY/$PTH/logging-curator:$VERSION $REGISTRY/$PTH/metrics-deployer:$VERSION $REGISTRY/$PTH/metrics-hawkular-metrics:$VERSION $REGISTRY/$PTH/metrics-cassandra:$VERSION $REGISTRY/$PTH/metrics-heapster:$VERSION

8.3 下载 apps 镜像
docker pull $REGISTRY/jboss-webserver-3/webserver30-tomcat7-openshift:latest; \
docker pull $REGISTRY/jboss-webserver-3/webserver30-tomcat8-openshift:latest; \
docker pull $REGISTRY/jboss-eap-6/eap64-openshift:latest; \
docker pull $REGISTRY/jboss-eap-7/eap70-openshift:latest; \

docker pull $REGISTRY/jboss-amq-6/amq62-openshift:latest; \
docker pull $REGISTRY/jboss-fuse-6/fis-java-openshift:latest; \
docker pull $REGISTRY/jboss-fuse-6/fis-karaf-openshift:latest; \
docker pull $REGISTRY/jboss-processserver-6/processserver63-openshift:latest; \
docker pull $REGISTRY/jboss-decisionserver-6/decisionserver63-openshift:latest; \

docker pull $REGISTRY/rhscl/mongodb-32-rhel7:latest; \
docker pull $REGISTRY/rhscl/mysql-56-rhel7:latest; \
docker pull $REGISTRY/rhscl/mysql-57-rhel7:latest; \
docker pull $REGISTRY/rhscl/php-56-rhel7:latest; \
docker pull $REGISTRY/rhscl/php-70-rhel7:latest; \
docker pull $REGISTRY/rhscl/python-35-rhel7:latest; \
docker pull $REGISTRY/rhscl/redis-32-rhel7:latest; \
docker pull $REGISTRY/rhscl/ruby-23-rhel7:latest; \
docker pull $REGISTRY/rhscl/s2i-base-rhel7:latest; \

docker pull $REGISTRY/$PTH/jenkins-1-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-2-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-slave-base-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-slave-maven-rhel7:latest; \
docker pull $REGISTRY/$PTH/jenkins-slave-nodejs-rhel7:latest; \
docker pull $REGISTRY/$PTH/nodejs-010-rhel7:latest;

导出 apps 镜像为 tar.gz
docker save -o ose-images-apps-latest_`date +'%Y%m%d'`.tar.gz $REGISTRY/jboss-webserver-3/webserver30-tomcat8-openshift:latest $REGISTRY/jboss-eap-7/eap70-openshift:latest $REGISTRY/rhscl/mongodb-32-rhel7:latest $REGISTRY/rhscl/mysql-56-rhel7:latest $REGISTRY/rhscl/php-56-rhel7:latest $REGISTRY/rhscl/python-35-rhel7:latest $REGISTRY/rhscl/redis-32-rhel7:latest $REGISTRY/rhscl/ruby-23-rhel7:latest $REGISTRY/$PTH/jenkins-1-rhel7:latest $REGISTRY/$PTH/jenkins-2-rhel7:latest $REGISTRY/$PTH/nodejs-010-rhel7:latest;

导出 extra apps 镜像为 tar.gz
docker save -o ose-images-apps-extra-latest_`date +'%Y%m%d'`.tar.gz $REGISTRY/jboss-webserver-3/webserver30-tomcat7-openshift:latest $REGISTRY/jboss-eap-6/eap64-openshift:latest $REGISTRY/jboss-amq-6/amq62-openshift:latest $REGISTRY/jboss-processserver-6/processserver63-openshift:latest $REGISTRY/jboss-decisionserver-6/decisionserver63-openshift:latest $REGISTRY/rhscl/mysql-57-rhel7:latest $REGISTRY/rhscl/php-70-rhel7:latest $REGISTRY/rhscl/s2i-base-rhel7:latest $REGISTRY/$PTH/jenkins-slave-base-rhel7:latest $REGISTRY/$PTH/jenkins-slave-maven-rhel7:latest;