本文參考openshift 3.11離線安裝鞍陨，遇到了一些環(huán)境相關(guān)的問(wèn)題噪猾，所以梳理了以下，如果遇到類(lèi)似問(wèn)題脐湾，可以參考。
環(huán)境準(zhǔn)備
- 硬件環(huán)境
公司內(nèi)部虛擬機(jī)3臺(tái) host-10-1-236-72, host-10-1-236-73, host-10-1-236-77
操作系統(tǒng) ：CentOS 7.4
[root@host-10-1-236-72 ~]# cat /etc/redhat-release
CentOS Linux release 7.4.1708 (Core)
- 手動(dòng)配置/etc/hosts
[root@host-10-1-236-72 ~]# cat /etc/hosts
10.1.236.72 host-10-1-236-72
10.1.236.73 host-10-1-236-73
10.1.236.77 host-10-1-236-77
- 節(jié)點(diǎn)安排
- master節(jié)點(diǎn)：host-10-1-236-72
- infra節(jié)點(diǎn)：host-10-1-236-73
- compute節(jié)點(diǎn)：host-10-1-236-77
- 在所有節(jié)點(diǎn)上設(shè)置selinux
[root@host-10-1-236-72 ~]# cat /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=enforcing
#SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected.
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
- 離線安裝必須要準(zhǔn)備安裝中依賴(lài)的yum源
源標(biāo)識(shí) 源名稱(chēng) 狀態(tài)
atomic atomic 12
centos-openshift-origin311 CentOS OpenShift Origin 60
ceph-common ceph-common 14
container-selinux container-selinux 18
docker docker 17
epel epel 1
extras extras 434
python-docker python-docker 4
- 節(jié)點(diǎn)間設(shè)置ssh免密訪問(wèn)
ssh-keygen -f ~/.ssh/id_rsa -N ''
for host in host-10-1-236-73 host-10-1-236-77
do
ssh-copy-id -i ~/.ssh/id_rsa.pub $host;
done
- 安裝iptables叙淌，并配置iptables
yum install iptables-services -y
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 81 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;systemctl enable iptables
Tips
由于Opshift 使用管理節(jié)點(diǎn)的8443端口沥割，因此除了上述端口加入iptables外耗啦，還需要將8443端口加入到iptables，否則在安裝后加入nodes時(shí)机杜，無(wú)法加入帜讲。
-A INPUT -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT
-A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT
- 找一臺(tái)機(jī)器作為ansible執(zhí)行的機(jī)器，安裝openshift ansible
yum install openshift-ansible
- 配置ansile hosts
[root@host-10-1-236-77 ~]# cat /etc/ansible/hosts
# Create an OSEv3 group that contains the masters, nodes, and etcd groups
[OSEv3:children]
masters
nodes
etcd
# Set variables common for all OSEv3 hosts
[OSEv3:vars]
# SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root
#openshift_deployment_type=openshift-enterprise
openshift_deployment_type=origin
openshift_release="3.11"
openshift_image_tag=v3.11
openshift_pkg_version=-3.11.0
openshift_use_openshift_sdn=true
# If ansible_ssh_user is not root, ansible_become must be set to true
#ansible_become=true
#containerized=false
# default selectors for router and registry services
# openshift_router_selector='node-role.kubernetes.io/infra=true'
# openshift_registry_selector='node-role.kubernetes.io/infra=true'
# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
#openshift_master_default_subdomain=ai.com
openshift_disable_check=memory_availability,disk_availability,docker_image_availability
os_sdn_network_plugin_name='redhat/openshift-ovs-networkpolicy'
openshift_master_cluster_method=native
openshift_master_cluster_hostname=host-10-1-236-72
openshift_master_cluster_public_hostname=host-10-1-236-72
# false
ansible_service_broker_install=false
openshift_enable_service_catalog=false
template_service_broker_install=false
openshift_logging_install_logging=false
enable_excluders=false
# registry passwd
#oreg_url=10.1.236.77:5000/openshift3/ose-${component}:${version}
oreg_url=10.1.236.77:5000/openshift/origin-${component}:${version}
openshift_examples_modify_imagestreams=true
# docker config
openshift_docker_additional_registries=10.1.236.77:5000
openshift_docker_insecure_registries=10.1.236.77:5000
#openshift_docker_blocked_registries
openshift_docker_options="--log-driver json-file --log-opt max-size=1M --log-opt max-file=3"
# openshift_cluster_monitoring_operator_install=false
# openshift_metrics_install_metrics=true
# openshift_enable_unsupported_configurations=True
#openshift_logging_es_nodeselector='node-role.kubernetes.io/infra: "true"'
#openshift_logging_kibana_nodeselector='node-role.kubernetes.io/infra: "true"'
# host group for masters
[masters]
host-10-1-236-72
# host group for etcd
[etcd]
host-10-1-236-72
# host group for nodes, includes region info
[nodes]
host-10-1-236-72 openshift_node_group_name='node-config-master'
host-10-1-236-73 openshift_node_group_name='node-config-compute'
host-10-1-236-77 openshift_node_group_name='node-config-compute'
host-10-1-236-73 openshift_node_group_name='node-config-infra'
- 安裝docker
yum install docker
- 找一臺(tái)機(jī)器作為docker鏡像倉(cāng)庫(kù)椒拗，安裝docker-distribution
本環(huán)境選 host-10-1-236-77節(jié)點(diǎn)安裝docker鏡像倉(cāng)庫(kù)
yum install -y docker-distribution;systemctl start docker-distribution;systemctl enable docker-distribution
- 配置docker /etc/containers/registries.conf似将，修改鏡像倉(cāng)庫(kù)地址。分發(fā)到各節(jié)點(diǎn)并重啟docker
[root@host-10-1-236-77 ~]# cat /etc/containers/registries.conf
# This is a system-wide configuration file used to
# keep track of registries for various container backends.
# It adheres to TOML format and does not support recursive
# lists of registries.
# The default location for this configuration file is /etc/containers/registries.conf.
# The only valid categories are: 'registries.search', 'registries.insecure',
# and 'registries.block'.
[registries.search]
registries = ['10.1.236.77:5000']
# If you need to access insecure registries, add the registry's fully-qualified name.
# An insecure registry is one that does not have a valid SSL certificate or only does HTTP.
[registries.insecure]
registries = ['10.1.236.77:5000']
# If you need to block pull access from a registry, uncomment the section below
# and add the registries fully-qualified name.
#
# Docker only
[registries.block]
registries = [registry.access.redhat.com]
ansible all -m copy -a "src=/etc/containers/registries.conf dest=/etc/containers/registries.conf "
ansible all -a 'systemctl start docker';ansible all -a 'systemctl enable docker'
- 下載如下docker鏡像蚀苛，并上傳到私有鏡像倉(cāng)庫(kù)中.(離線安裝使用)
[root@host-10-1-236-77 data]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
docker.io/openshift/origin-node v3.11 32492bc114ab 4 weeks ago 1.16 GB
docker.io/openshift/origin-control-plane v3.11 bf9dd3be958f 4 weeks ago 818 MB
docker.io/openshift/origin-deployer v3.11.0 c3351a73e321 4 weeks ago 376 MB
docker.io/openshift/origin-haproxy-router v3.11 88f17a56a362 4 weeks ago 401 MB
docker.io/openshift/origin-pod v3.11.0 947fb5afa0ce 4 weeks ago 254 MB
docker.io/openshift/origin-web-console v3.11 be30b6cce5fa 6 weeks ago 339 MB
docker.io/openshift/origin-docker-registry v3.11 a0d5ad164395 6 weeks ago 305 MB
docker.io/openshift/origin-metrics-server v3.11 8c99f32f40d3 6 weeks ago 311 MB
docker.io/openshift/origin-console v3.11 c2b807836120 6 weeks ago 260 MB
docker.io/openshift/origin-metrics-heapster v3.11 69421c019449 6 weeks ago 710 MB
docker.io/openshift/origin-metrics-hawkular-metrics v3.11 59e2258250c4 6 weeks ago 860 MB
docker.io/openshift/origin-metrics-schema-installer v3.11 342f50fded7d 6 weeks ago 551 MB
docker.io/openshift/origin-metrics-cassandra v3.11 8176cfabc16b 6 weeks ago 590 MB
docker.io/cockpit/kubernetes latest 4ec8c5b8a486 6 weeks ago 336 MB
quay.io/coreos/cluster-monitoring-operator v0.1.1 4488a207a5bc 2 months ago 510 MB
quay.io/coreos/prometheus-config-reloader v0.23.2 2ed5973a47af 3 months ago 12.2 MB
quay.io/coreos/prometheus-operator v0.23.2 835a7e260b35 3 months ago 47 MB
docker.io/openshift/prometheus-alertmanager v0.15.2 68bbd0006378 3 months ago 233 MB
docker.io/openshift/prometheus-node-exporter v0.16.0 f9f775bf6d0e 4 months ago 216 MB
docker.io/openshift/prometheus v2.3.2 e362c322f000 4 months ago 316 MB
docker.io/grafana/grafana 5.2.1 1bfead9ff707 5 months ago 245 MB
quay.io/coreos/kube-rbac-proxy v0.3.1 992ac1a5e7c7 5 months ago 40.2 MB
quay.io/coreos/etcd v3.2.22 ff5dd2137a4f 5 months ago 37.3 MB
quay.io/coreos/kube-state-metrics v1.3.1 a9c8f313b7aa 7 months ago 22.2 MB
docker.io/openshift/oauth-proxy v1.1.0 90c45954eb03 9 months ago 235 MB
quay.io/coreos/configmap-reload v0.0.1 3129a2ca29d7 23 months ago 4.79 MB
給鏡像打私有倉(cāng)庫(kù)的tag并push到私有倉(cāng)庫(kù)中在验。
docker images|grep docker.io/openshift/origin-node|awk '{print $1"/"$2}'|awk -F "/" '{print "docker tag "$1"/"$2"/"$3":"$4" 10.1.236.77:5000/"$2"/"$3":"$4}'|sh
docker images|grep 10.1.236.77:5000|awk '{print "docker push " $1":"$2}'|sh
- 使用ansible book預(yù)檢查openshift
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml
Tips
- 在預(yù)檢查過(guò)程中，如果報(bào)錯(cuò)堵未，可以通過(guò)jourctl查看錯(cuò)誤原因腋舌，如果是因?yàn)殓R像不全，需要按照#13步操作方式渗蟹，離線下載缺少的鏡像并上傳到私有倉(cāng)庫(kù)
- 預(yù)檢查其中有一個(gè)task會(huì)自動(dòng)將CentOS-OpenShift-Origin.repo更新為公網(wǎng)的repo块饺，會(huì)將私有repo替換掉，導(dǎo)致預(yù)檢查失敗雌芽。在檢查中授艰，可以手動(dòng)替換，這樣retry就會(huì)過(guò)去世落。
- 使用ansible book安裝openshift
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml -vvv
- 如果安裝過(guò)程中出現(xiàn)問(wèn)題淮腾，也可以通過(guò)ansible清除下，再重新安裝屉佳，命令如下谷朝。
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml
Tips
刪除后，必須要手動(dòng)將ip link刪除武花，否則再次安裝時(shí)徘禁，安裝失敗，報(bào)錯(cuò)"OKD install fails: network plugin is not ready: cni config uninitialize on default SDN"
查看SDN pods日志SDN pod拉起時(shí)就會(huì)報(bào)錯(cuò). "SDN node startup failed: node SDN setup failed: error on port vxlan0: "could not add network device vxlan0 to ofproto (File exists)"
遇到這種問(wèn)題時(shí)髓堪，在節(jié)點(diǎn)上使用ip link來(lái)查詢并刪除，操作命令如下：
ip link delete vxlan_sys_4789
- 安裝結(jié)束后創(chuàng)建用戶
htpasswd -cb /etc/origin/master/htpasswd admin mypwd
oc adm policy add-cluster-role-to-user cluster-admin admin
- 部署后檢查
- 后臺(tái)操作檢查
檢查節(jié)點(diǎn)是否ready
[root@host-10-1-236-72 ~]# oc get nodes
NAME STATUS ROLES AGE VERSION
host-10-1-236-72 Ready master 6d v1.11.0+d4cacc0
host-10-1-236-73 Ready infra 6d v1.11.0+d4cacc0
host-10-1-236-77 Ready compute 6d v1.11.0+d4cacc0
檢查projects
[root@host-10-1-236-72 ~]# oc projects
You have access to the following projects and can switch between them with 'oc project <projectname>':
* default
kube-public
kube-system
management-infra
openshift
openshift-console
openshift-infra
openshift-logging
openshift-monitoring
openshift-node
openshift-sdn
openshift-web-console
- 前臺(tái)操作檢查
瀏覽器打開(kāi)鏈接https://host-10-1-236-72:8443 出現(xiàn)登錄界面娘荡。login
使用admin用戶登錄系統(tǒng)
dashboard
Tips
oc logs無(wú)法查看pods日志干旁，報(bào)錯(cuò)："ipaddress:10250: connect: no route to host"。原因在于需要放開(kāi)子節(jié)點(diǎn)10250端口的iptables炮沐。操作如下争群，檢查/etc/sysconfig/iptables，如果10250端口沒(méi)有被加進(jìn)去大年，添加10250的策略
-A INPUT -p tcp -m state --state NEW -m tcp --dport 10250 -j ACCEPT
-A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 10250 -j ACCEPT
