kubeadm部署高可用
堆疊ETCD模型 這種方法需要更少的基礎(chǔ)架構(gòu)哺眯。etcd成員和控制平面節(jié)點(diǎn)位于同一位置糯俗。
https://kubernetes.io/zh/docs/setup/production-environment/tools/kubeadm/high-availability/
ETCD 是一個(gè)高可用的分布式鍵值數(shù)據(jù)庫(kù)屈梁，可用于服務(wù)發(fā)現(xiàn)它褪。ETCD 采用 raft 一致性算法旬迹，基于 Go 語(yǔ)言實(shí)現(xiàn)弄贿。etcd作為一個(gè)高可用鍵值存儲(chǔ)系統(tǒng)沟蔑，天生就是為集群化而設(shè)計(jì)的湿诊。由于Raft算法在做決策時(shí)需要多數(shù)節(jié)點(diǎn)的投票，所以etcd一般部署集群推薦奇數(shù)個(gè)節(jié)點(diǎn)瘦材，推薦的數(shù)量為3厅须、5或者7個(gè)節(jié)點(diǎn)構(gòu)成一個(gè)集群。
配置三臺(tái)機(jī)器 kubeadm 的最低要求給主節(jié)點(diǎn)
| 角色 | IP |
|---|---|
| master1 | 10.0.0.2 |
| master2 | 10.0.0.3 |
| master3 | 10.0.0.4 |
| VIP（虛擬ip） | 10.0.0.10 |
1.haproxy+keepalived
構(gòu)建haproxy容器
cat >Dockerfile<<\EOF
FROM alpine:3.7
RUN apk add tzdata \
&& cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
RUN apk add --no-cache haproxy
CMD ["haproxy","-f","/etc/haproxy/haproxy.cfg"]
EOF
docker build . -t haproxy:v1
主備啟用haproxy容器
mkdir -p /usr/local/etc/haproxy/
cat >/usr/local/etc/haproxy/haproxy.cfg<<\EOF
global
#log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 65535
chroot /var/lib/haproxy
user haproxy
group haproxy
#daemon
defaults
log global
mode tcp
option httplog
option dontlognull
retries 3
option redispatch
timeout queue 1m
timeout connect 50s
timeout client 50s
timeout server 50s
timeout check 10s
maxconn 102400
listen stats
stats enable
bind *:8081
mode http
option httplog
log global
maxconn 10
stats refresh 30s
stats uri /haproxy/stats
stats auth haproxy:haproxy9527
stats hide-version
# stats admin if TRUE
frontend fe_k8s_6444
bind *:6444
mode tcp
log global
option tcplog
default_backend be_k8s_6443
backend be_k8s_6443
mode tcp
balance roundrobin
server k8s-master01 10.0.0.2:6443 maxconn 4096 check weight 1 check inter 6s fall 3 rise 3
server k8s-master02 10.0.0.3:6443 maxconn 4096 check weight 1 check inter s fall 3 rise 3
EOF
docker run -d --name haproxy \
--restart=always \
--net=host \
-v /usr/local/etc/haproxy/haproxy.cfg:/etc/haproxy/haproxy.cfg \
haproxy:v1
平滑重啟
docker kill -s HUP haproxy
構(gòu)建keepalived容器
cat >Dockerfile<<\EOF
FROM alpine:3.7
RUN apk add tzdata \
&& cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
RUN apk add --no-cache keepalived
CMD ["keepalived","-f","/etc/keepalived/keepalived.conf","-P","-l","-n"]
EOF
docker build . -t keepalived:v1
主備節(jié)點(diǎn)啟動(dòng)keepalive
mkdir -p /usr/local/etc/keepalived
cat >/usr/local/etc/keepalived/keepalived.conf<<EOF
global_defs {
router_id lb01
}
vrrp_script chk_haproxy {
script "/bin/busybox nc -v -w 2 -z 127.0.0.1 6444 2>&1 | grep open"
timeout 1
interval 1 # check every 1 second
fall 2 # require 2 failures for KO
rise 2 # require 2 successes for OK
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 50
priority 100
advert_int 1
nopreempt # 不搶占
track_script {
chk_haproxy
}
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
10.0.0.10/24 dev ens33
}
}
EOF
# state MASTER或BUCKUP
# priority 權(quán)重,備節(jié)點(diǎn)權(quán)重設(shè)置90食棕，備要小于主節(jié)點(diǎn)
# 未設(shè)置weight時(shí)朗和，weight默認(rèn)值為0，此時(shí)當(dāng)vrrp_script連續(xù)檢測(cè)失敗時(shí)簿晓，vrrp實(shí)例進(jìn)入FAULT狀態(tài)眶拉。會(huì)導(dǎo)致VIP轉(zhuǎn)移
# nopreempt 不搶占，當(dāng)節(jié)點(diǎn)不可用時(shí)憔儿，才漂移
docker run -d --name keepalived \
--restart=always \
--net=host --cap-add=NET_ADMIN \
-v /usr/local/etc/keepalived/keepalived.conf:/etc/keepalived/keepalived.conf \
keepalived:v1
2. kubeadm部署
要對(duì)kubeadm源代碼修改 做證書(shū)年限更改的操作忆植，所有機(jī)器
master1
mkdir /usr/local/kubernetes/manifests -p
cd /usr/local/kubernetes/manifests/
kubeadm config print init-defaults > kubeadm-config.yaml
#更改以下內(nèi)容
advertiseAddress: 10.0.0.2
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kubernetesVersion: v1.19.2
podSubnet: 192.168.0.0/16
serviceSubnet: 10.96.0.0/12
#結(jié)尾添加：
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
#在clusterName: kubernetes：下添加
controlPlaneEndpoint: "10.0.0.10:6444"
kubeadm init --config=kubeadm-config.yaml --upload-certs | tee kubeadm-init.log
wget https://kuboard.cn/install-script/calico/calico-3.13.1.yaml
kubectl apply -f calico-3.13.1.yaml
master2,master3
kubeadm join 10.0.0.10:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:01c9edd09d8838d0b12ba6e043b72cb4f6814613ef5b857b62d6645b4397a11a \
--control-plane --certificate-key fd5e0e7c3e78d718aa0c935e46423f96eb8420231ead56e4e5070c5090f0606a
node節(jié)點(diǎn)
kubeadm join 10.0.0.10:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:01c9edd09d8838d0b12ba6e043b72cb4f6814613ef5b857b62d6645b4397a11a
檢查群集運(yùn)行狀況
kubectl get pods --all-namespaces
docker run --rm -it \
--net host \
-v /etc/kubernetes:/etc/kubernetes 0369cf4303ff etcdctl \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://10.0.0.2:2379 endpoint health --cluster
kubectl get endpoints kube-controller-manager -n kube-system -o yaml
kubectl get endpoints kube-scheduler -n kube-system -o yaml
3.kubeadm的etcd備份恢復(fù)
yum -y install etcd
cd /data/etcd/
# 在每個(gè)節(jié)點(diǎn)備份
ETCDCTL_API=3 etcdctl snapshot save snap.db \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/peer.crt \
--key=/etc/kubernetes/pki/etcd/peer.key
# 查看備份
ETCDCTL_API=3 etcdctl snapshot status snap.db
# 查看現(xiàn)有成員狀態(tài)
ETCDCTL_API=3 etcdctl \
--endpoints='10.0.0.2:2379,10.0.0.3:2379,10.0.0.4:2379' \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/peer.crt \
--key=/etc/kubernetes/pki/etcd/peer.key \
--write-out="table" \
endpoint status
# 剔除
ETCDCTL_API=3 etcdctl \
--endpoints='10.0.0.2:2379,10.0.0.3:2379,10.0.0.4:2379' \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/peer.crt \
--key=/etc/kubernetes/pki/etcd/peer.key \
member remove a705d171b1d0bf1c
#
# member add k8s-slave02 --peer-urls=http://10.0.0.4:2380
# member list
恢復(fù)
#1.先暫停kube-apiserver和etcd容器
mv /etc/kubernetes/manifests /etc/kubernetes/manifests.bak
mv /var/lib/etcd/ /var/lib/etcd.bak
#2.恢復(fù)
ETCDCTL_API=3 etcdctl \
snapshot restore snap.db \
--data-dir=/var/lib/etcd
#3.啟動(dòng)kube-apiserver和etcd容器
mv /etc/kubernetes/manifests.bak /etc/kubernetes/manifests
4.dashboard
mkdir /data/k8s-yaml/dashboard/ && cd /data/k8s-yaml/dashboard/
# 需要網(wǎng)絡(luò)可以訪問(wèn)
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.4/aio/deploy/recommended.yaml
# 查看是否已經(jīng)在運(yùn)行了
kubectl get pod -n kubernetes-dashboard
# 新建sa，綁定權(quán)限
kubectl create sa -name admin-user -n kubernetes-dashboard
kubectl create clusterrolebinding admin-user --clusterrole=cluster-admin \
--serviceaccount=kubernetes-dashboard:admin-user
# 獲取token
kubectl -n kubernetes-dashboard describe secret \
$(kubectl -n kubernetes-dashboard get secret | grep admin-user | awk '{print $1}')
#1. 編輯kubernetes-dashboard皿曲，將里面的type: ClusterIP改為type: NodePort即可
#kubectl --namespace=kubernetes-dashboard edit service kubernetes-dashboard
#2.通過(guò)ingrees訪問(wèn)
通過(guò)ingress訪問(wèn)
cat dash-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kubernetes-dashboard
annotations:
kubernetes.io/ingress.class: "nginx"
# 開(kāi)啟use-regex唱逢，啟用path的正則匹配
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/rewrite-target: /
# 默認(rèn)為 true，啟用 TLS 時(shí)屋休，http請(qǐng)求會(huì) 308 重定向到https
nginx.ingress.kubernetes.io/ssl-redirect: "true"
# 默認(rèn)為 http坞古，開(kāi)啟后端服務(wù)使用 proxy_pass https://協(xié)議
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
tls:
- hosts:
- dash.zs.com
secretName: kubernetes-dashboard-certs
rules:
- host: dash.zs.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: kubernetes-dashboard
port:
number: 443
# 簡(jiǎn)單的方法
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kubernetes-dashboard
annotations:
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
rules:
- host: dash.zs.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: kubernetes-dashboard
port:
number: 443
