硬件准备

初始化配置

以下如无特殊说明则只安装master节点

Hosts配置

[root@localhost ~]# vim /etc/hosts
192.168.30.171 master-test-k8s
192.168.30.170 node1-test-k8s
192.168.30.172 node2-test-k8s
192.168.30.173 node3-test-k8s
192.168.30.174 node4-test-k8s
192.168.30.175 infra1-test-k8s
192.168.30.176 infra2-test-k8s

修改主机名(所有节点)

[root@localhost ~]# hostnamectl set-hostname master-test-k8s

关闭防火墙

# 如果安装过程中有提示，则修改对应的SELINUX，否则不需要修改
[root@localhost ~]# vim /etc/selinux/config
SELINUX=permissive
SELINUXTYPE=targeted

# 关闭防火墙
[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld

免密钥登录

[root@localhost ~]# ssh-keygen -b 4096 -f ~/.ssh/id_rsa -N ""
[root@localhost ~]# cat ~/.ssh/id_rsa.pub | tee -a ~/.ssh/authorized_keys
[root@localhost ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub <user>@<node_ip_address>

安装基础包

[root@localhost ~]# yum install wget git net-tools bind-utils yum-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct glusterfs-fuse -y
[root@localhost ~]# yum update
[root@localhost ~]# reboot

到这一步出问题了，reboot之后一直连不上服务器了，原因是SELINUX=enforcing之后导致的。

安装docker(所有节点都安装)

[root@localhost ~]# yum install docker-1.13.1 -y
[root@localhost ~]# rpm -V docker-1.13.1
[root@localhost ~]# docker version
# systemctl enable docker 设置docker开机启动
[root@localhost ~]# systemctl start docker

安装 epel 套件

[root@localhost ~]# yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
[root@localhost ~]# yum -y --enablerepo=epel install pyOpenSSL

安装指定版本的ansible

[root@localhost ~]# yum --showduplicates list ansible
[root@localhost ~]# yum install ansible-2.7.4-1.el7ae -y
[root@localhost ~]# ansible --version
ansible 2.9.6
  config file = /etc/ansible/ansible.cfg
  configured module search path = [u'/root/.ansible/plugins/modules', u'/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/lib/python2.7/site-packages/ansible
  executable location = /usr/bin/ansible
  python version = 2.7.5 (default, Aug  7 2019, 00:51:29) [GCC 4.8.5 20150623 (Red Hat 4.8.5-39)]

准备高可用部署

# 在两台infra节点上进行
[root@localhost ~]# yum -y install keepalived

# 第一个infra节点
# 直接删除原内容 使用下边的替换
[root@localhost ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER
    interface enp3s0
    virtual_router_id 55
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.30.176
    }
}

# 第二个infra节点
[root@localhost ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state BACKUP
    interface enp3s0
    virtual_router_id 55
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.30.176
    }
}
# 在两个基础设施节点上都启用并启动 Keepalived:
[root@localhost ~]# systemctl enable keepalived
[root@localhost ~]# systemctl start keepalived
[root@localhost ~]# journalctl -u keepalived

# 查看运行状态
[root@infra1 ~]# ps -ef|grep keep
root      1635     1  0 14:58 ?        00:00:00 /usr/sbin/keepalived -D
root      1636  1635  0 14:58 ?        00:00:00 /usr/sbin/keepalived -D
root      1637  1635  0 14:58 ?        00:00:00 /usr/sbin/keepalived -D
root      1693  1589  0 14:59 pts/0    00:00:00 grep --color=auto keep
[root@infra1 ~]# ip address show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp3s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 34:97:f6:87:6f:a3 brd ff:ff:ff:ff:ff:ff
    inet 192.168.30.175/22 brd 192.168.31.255 scope global noprefixroute enp3s0
       valid_lft forever preferred_lft forever
    inet 192.168.30.176/32 scope global enp3s0
       valid_lft forever preferred_lft forever
    inet6 fe80::727c:db96:f863:82c1/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

OpenShift安装配置文件准备

# 使用 Openshift-ansible 安裝方式
[root@master-test-k8s ~]# cd ~
[root@master-test-k8s ~]# git clone https://github.com/openshift/openshift-ansible
[root@master-test-k8s ~]# cd openshift-ansible
[root@master-test-k8s ~]# git checkout release-3.11

[root@master-test-k8s ~]# pwd
/root/openshift-ansible/inventory
[root@master-test-k8s ~]# cp hosts.example hosts

# 编辑配置文件
[masters]
master-test-k8s

[etcd]
master-test-k8s

[nodes]
master-test-k8s openshift_node_group_name="node-config-master"
infra1-test-k8s openshift_node_group_name="node-config-infra"
infra2-test-k8s openshift_node_group_name="node-config-infra"
node1-test-k8s openshift_node_group_name="node-config-compute"
node2-test-k8s openshift_node_group_name="node-config-compute"
node3-test-k8s openshift_node_group_name="node-config-compute"
node4-test-k8s openshift_node_group_name="node-config-compute"

[nfs]
master-test-k8s

[lb]
infra1-test-k8s
infra2-test-k8s

# Create an OSEv3 group that contains the masters and nodes groups
[OSEv3:children]
masters
nodes
etcd
lb
nfs

[OSEv3:vars]
ansible_user=root

openshift_deployment_type=origin
openshift_release="3.11"

openshift_master_default_subdomain=infra1-test-k8s
openshift_master_cluster_method=native

openshift_master_cluster_hostname=infra1-test-k8s
openshift_master_cluster_public_hostname=infra1-test-k8s

debug_level=2

openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
openshift_disable_check=memory_availability,disk_availability,docker_storage,docker_image_availability

openshift_hosted_registry_storage_kind=nfs
openshift_hosted_registry_storage_access_modes=['ReadWriteMany']
openshift_hosted_registry_storage_nfs_directory=/exports
openshift_hosted_registry_storage_nfs_options='*(rw,root_squash)'
openshift_hosted_registry_storage_volume_name=registry
openshift_hosted_registry_storage_volume_size=200Gi
openshift_clock_enabled=true
openshift_enable_unsupported_configurations=True

DNS下发(Master节点执行)

# 将hosts复制到/etc/ansible/下
[root@master-test-k8s ~]# cp ~/openshift-ansible/inventory/hosts /etc/ansible/hosts
[root@master-test-k8s ~]# ansible all -m copy -a "src=/etc/hosts dest=/etc/hosts "

Docker启动(Master节点执行)

[root@master-test-k8s ~]# ansible all -a 'systemctl start docker';ansible all -a 'systemctl enable docker'

执行安装(Master节点执行)

# 理论上 下边两条ansible-playbook的命令是可以反复执行的
[root@master-test-k8s ~]# cd openshift-ansible/
[root@master-test-k8s openshift-ansible]# ansible-playbook -i inventory/hosts playbooks/prerequisites.yml
[root@master-test-k8s openshift-ansible]# ansible-playbook -i inventory/hosts playbooks/deploy_cluster.yml

# 部署结束后, 创建用户
[root@master-test-k8s openshift-ansible]# htpasswd -cb /etc/origin/master/htpasswd admin admin

# OpenShift卸载命令:
[root@master-test-k8s openshift-ansible]# ansible-playbook ./playbooks/adhoc/uninstall.yml