一、服務器環(huán)境及部署圖

• 操作系統(tǒng) CentOS 7.6
• 服務器類型,運行組件及配置如下
  10.2.2.10 2C/2G MySQL 5.7 容器方式運行,Rancher UI (用于集群存儲及提供ui界面)
  10.2.2.11 2C/512M Haproxy-1,keepalived-1 (用于集群負載均衡及高可用)
  10.2.2.12 2C/512M Haproxy-2,keepalived-2 (用于集群負載均衡及高可用)
  10.2.2.13 2C/2G k3s-server-1
  10.2.2.14 2C/2G k3s-server-2
  10.2.2.15 2C/2G k3s-agent-1
  10.2.2.16 2C/2G k3s-agent-2
  10.2.2.100 Keepalived-VIP (虛擬VIP它呀，并非服務器)

• 部署圖如下

  
  
  image.png

二蹲缠、服務器初始化腳本

執(zhí)行范圍：所有主機
執(zhí)行完成后重啟服務器校赤。

#!/bin/sh
# 服務器初始化腳本(所有主機執(zhí)行)

# 設(shè)置當前主機ip地址環(huán)境（帶d的為開發(fā)用司澎，帶p的為預發(fā)布用粥烁，帶t的為測試用，帶a的為生產(chǎn)用）
IP_ENV=t

# 關(guān)閉防火墻现斋、selinux
sudo systemctl stop firewalld && sudo systemctl disable firewalld
sudo systemctl stop iptables && sudo systemctl disable iptables
sudo setenforce 0
sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
sudo sed -i 's/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config

# 禁用swap
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# 關(guān)閉networkmanager
sudo systemctl stop NetworkManager && sudo systemctl disable NetworkManager

# 修改時區(qū)語言
sudo ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
sudo sh -c 'echo 'LANG="en_US.UTF-8"' >> /etc/profile'
source /etc/profile

# 修改最大文件打開數(shù)
# sudo sh -c ''
sudo sh -c 'echo '* soft nofile 65535' >> /etc/security/limits.conf'
sudo sh -c 'echo '* hard nofile 65535' >> /etc/security/limits.conf'

# 啟用ipvs內(nèi)核模塊
sudo /sbin/modprobe ip_vs_dh
sudo /sbin/modprobe ip_vs_fo
sudo /sbin/modprobe ip_vs_ftp
sudo /sbin/modprobe ip_vs
sudo /sbin/modprobe ip_vs_lblc
sudo /sbin/modprobe ip_vs_lblcr
sudo /sbin/modprobe ip_vs_lc
sudo /sbin/modprobe ip_vs_nq
sudo /sbin/modprobe ip_vs_ovf
sudo /sbin/modprobe ip_vs_pe_sip
sudo /sbin/modprobe ip_vs_rr
sudo /sbin/modprobe ip_vs_sed
sudo /sbin/modprobe ip_vs_sh
sudo /sbin/modprobe ip_vs_wlc
sudo /sbin/modprobe ip_vs_wrr

# 將橋接的IPv4流量傳遞到iptables的鏈：
# 如果有配置，則修改
sudo sed -i "s#^net.ipv4.ip_forward.*#net.ipv4.ip_forward=1#g"  /etc/sysctl.conf
sudo sed -i "s#^net.bridge.bridge-nf-call-ip6tables.*#net.bridge.bridge-nf-call-ip6tables=1#g"  /etc/sysctl.conf
sudo sed -i "s#^net.bridge.bridge-nf-call-iptables.*#net.bridge.bridge-nf-call-iptables=1#g"  /etc/sysctl.conf
sudo sed -i "s#^net.ipv6.conf.all.disable_ipv6.*#net.ipv6.conf.all.disable_ipv6=1#g"  /etc/sysctl.conf
sudo sed -i "s#^net.ipv6.conf.default.disable_ipv6.*#net.ipv6.conf.default.disable_ipv6=1#g"  /etc/sysctl.conf
sudo sed -i "s#^net.ipv6.conf.lo.disable_ipv6.*#net.ipv6.conf.lo.disable_ipv6=1#g"  /etc/sysctl.conf
sudo sed -i "s#^net.ipv6.conf.all.forwarding.*#net.ipv6.conf.all.forwarding=1#g"  /etc/sysctl.conf
# 可能沒有偎蘸，追加
sudo sh -c 'echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.lo.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.all.forwarding = 1"  >> /etc/sysctl.conf'
# 此參數(shù)為elasticsearch運行需要調(diào)整的參數(shù)
sudo sh -c 'echo "vm.max_map_count = 655300" >> /etc/sysctl.conf' 
# 使得上述配置生效
sudo sysctl -p

# 修改主機名
# 獲取ip后兩組數(shù)字（帶d的為開發(fā)用庄蹋，帶p的為預發(fā)布用，帶t的為測試用迷雪，帶a的為生產(chǎn)用）
ipNumlast2=`ip addr|egrep '[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'|grep -v '127'|awk -F'[ ]+' '{print $3}'|cut -d / -f 1|cut -d . -f 3-4|tr "\." "${IP_ENV}"`
# 設(shè)置hostname
sudo hostnamectl set-hostname $ipNumlast2.cluster

# 修改yum源
# 我默認使用163的源
sudo rm -rf /etc/yum.repos.d/*
sudo sh -c 'cat > /etc/yum.repos.d/163.repo <<EOF
[base]
name=CentOS-$releasever - Base - 163.com
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
baseurl=http://mirrors.163.com/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=http://mirrors.163.com/centos/RPM-GPG-KEY-CentOS-7

#released updates
[updates]
name=CentOS-$releasever - Updates - 163.com
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates
baseurl=http://mirrors.163.com/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=http://mirrors.163.com/centos/RPM-GPG-KEY-CentOS-7

#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras - 163.com
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras
baseurl=http://mirrors.163.com/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=http://mirrors.163.com/centos/RPM-GPG-KEY-CentOS-7

#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus - 163.com
baseurl=http://mirrors.163.com/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.163.com/centos/RPM-GPG-KEY-CentOS-7
EOF'

# 安裝一些基本工具和軟件
sudo yum -y install wget vim lsof net-tools chrony

# 配置時間同步
sudo sed -i 's/^server.*iburst$//' /etc/chrony.conf
sudo sh -c 'cat >> /etc/chrony.conf <<EOF
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
server ntp3.aliyun.com iburst
server ntp4.aliyun.com iburst
server ntp5.aliyun.com iburst
server ntp6.aliyun.com iburst
server ntp7.aliyun.com iburst
EOF'
sudo systemctl start chronyd
sudo systemctl enable chronyd
sudo chronyc sources -v

三限书、將IP和對應的主機名寫入hosts文件

執(zhí)行范圍： 所有主機

sudo sh -c 'cat >>/etc/hosts<<EOF
10.2.2.10 2t10 2t10.cluster
10.2.2.11 2t11 2t11.cluster
10.2.2.12 2t12 2t12.cluster
10.2.2.13 2t13 2t13.cluster
10.2.2.14 2t14 2t14.cluster
10.2.2.15 2t15 2t15.cluster
10.2.2.16 2t16 2t16.cluster
EOF'

四、docker 的安裝與配置

安裝方式： 二進制離線安裝
執(zhí)行范圍： 所有主機

1. 下載docker二進制安裝包
https://download.docker.com/linux/static/stable/

2. 上傳到服務器并解壓縮
[demo@2t16 docker]$ ls # 查看
docker-20.10.9.tgz
[demo@2t16 docker]$ tar -xvf docker-20.10.9.tgz  # 解壓縮
docker/
docker/containerd-shim-runc-v2
docker/dockerd
docker/docker-proxy
docker/ctr
docker/docker
docker/runc
docker/containerd-shim
docker/docker-init
docker/containerd

3. 將docker二進制程序文件拷貝到指定位置
[demo@2t16 docker]$ sudo mv docker/* /usr/bin/  # 拷貝
[demo@2t16 docker]$ ls /usr/bin/docker*   # 查看
/usr/bin/docker  /usr/bin/dockerd  /usr/bin/docker-init  /usr/bin/docker-proxy

4. 創(chuàng)建配置文件
[demo@2t16 docker]$ sudo mkdir /etc/docker # 先創(chuàng)建一個配置文件目錄
[demo@2t16 docker]$ sudo sh -c 'cat >/etc/docker/daemon.json<<EOF  # 將配置寫入到文件,請刪掉這些注釋
{
    "oom-score-adjust": -1000,
    "data-root": "/home/qfsystem/docker-data", # docker數(shù)據(jù)目錄 自定義
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m",
        "max-file": "10"
    },
    "exec-opts": [
        "native.cgroupdriver=cgroupfs"
    ],
    "max-concurrent-downloads": 10,
    "max-concurrent-uploads": 10,
    "bip": "182.10.0.1/16", # 默認網(wǎng)段 自定義 
    "insecure-registries": [
        "0.0.0.0/0" # 私有倉庫地址
    ],
    "registry-mirrors": [
        "https://yd48ur9i.mirror.aliyuncs.com" # 鏡像加速器地址
    ],
    "storage-driver": "overlay2",
    "storage-opts": [
        "overlay2.override_kernel_check=true"
    ]
}
EOF'

5. 創(chuàng)建systemd啟動啟動腳本
sudo sh -c 'cat>/etc/systemd/system/docker.service<<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
ExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT
LimitNOFILE=infinity
LimitNPROC=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target
EOF'

6. 啟動并查看狀態(tài)
[demo@2t14 docker]$ sudo systemctl daemon-reload   # 使配置生效
[demo@2t14 docker]$ sudo systemctl start docker    # 啟動docker
[demo@2t14 docker]$ sudo systemctl enable docker  # 配置docker開機自啟
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /etc/systemd/system/docker.service.
[demo@2t14 docker]$ sudo systemctl status docker   # 查看docker運行狀態(tài)
● docker.service - Docker Application Container Engine
   Loaded: loaded (/etc/systemd/system/docker.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2021-10-15 23:24:23 CST; 15s ago
     Docs: https://docs.docker.com
 Main PID: 21710 (dockerd)
   CGroup: /system.slice/docker.service
           ├─21710 /usr/bin/dockerd
           └─21740 containerd --config /var/run/docker/containerd/containerd.toml --log-level info
... ...

7. 檢查docker配置是否生效
[demo@2t16 docker]$ sudo docker info
Client:
 Context:    default
 Debug Mode: false

Server:
 Containers: 0
  Running: 0
  Paused: 0
  Stopped: 0
 Images: 0
 Server Version: 20.10.9
 Storage Driver: overlay2
  Backing Filesystem: xfs
  Supports d_type: true
  Native Overlay Diff: true
  userxattr: false
 Logging Driver: json-file
 Cgroup Driver: systemd
 Cgroup Version: 1
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: inactive
 Runtimes: io.containerd.runc.v2 io.containerd.runtime.v1.linux runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: 5b46e404f6b9f661a205e28d59c982d3634148f8
 runc version: v1.0.2-0-g52b36a2d
 init version: de40ad0
 Security Options:
  seccomp
   Profile: default
 Kernel Version: 4.4.246-1.el7.elrepo.x86_64
 Operating System: CentOS Linux 7 (Core)
 OSType: linux
 Architecture: x86_64
 CPUs: 2
 Total Memory: 470.9MiB
 Name: 2t16.cluster.ydca
 ID: CRUJ:Y2DX:A3A5:MKKK:PJIC:QDFT:2NDO:XVQ5:APAW:RMSJ:4K5Z:KPXY
 Docker Root Dir: /home/demo/docker-data
 Debug Mode: false
 Registry: https://index.docker.io/v1/
 Labels:
 Experimental: false
 Insecure Registries:
  0.0.0.0/0
  127.0.0.0/8
 Registry Mirrors:
  https://yd48ur9i.mirror.aliyuncs.com/
 Live Restore Enabled: false
 Product License: Community Engine

五章咧、啟動一個5.7版本mysql數(shù)據(jù)庫容器

作用：用于為k3s提供存儲蔗包，k3s支持除etcd外的集群數(shù)據(jù)存儲方式
執(zhí)行范圍： 10.2.2.10 服務器主機
說明：選用5.7版本是因為該版本是rancher官方推薦。
第5步中慧邮，只需要對server節(jié)點所在的IP創(chuàng)建用戶并授權(quán)就可以了调限。

1. 拉取mysql5.7鏡像
[demo@2t10 ~]$ sudo docker pull mysql:5.7

2. 編寫啟動腳本
[demo@2t10 ~]$ cat >/home/demo/start-k3s-mysql.sh<<EOF
#!/bin/sh
set -x
set -e

sudo docker run \
--restart=always \
--name mysql-service \
-v /home/demo/k3s-mysql-data:/var/lib/mysql \
-p 13306:3306 \
-e MYSQL_ROOT_PASSWORD=root \
-d mysql:5.7 \
--character-set-server=utf8mb4 \
--collation-server=utf8mb4_general_ci \
--lower_case_table_names=1 \
--skip-name-resolve=1 \
--max_connections=1000 \
--wait_timeout=31536000 \
--interactive_timeout=31536000 \
--innodb_large_prefix=on \
--default-time-zone='+8:00'
EOF

3. 腳本賦權(quán)并運行腳本
[demo@2t10 ~]$ chmod +x start-k3s-mysql.sh
[demo@2t10 ~]$ ./start-k3s-mysql.sh 

4. 查看容器運行情況
[demo@2t10 ~]$ sudo docker ps -a
CONTAINER ID   IMAGE       COMMAND                  CREATED              STATUS          PORTS                                NAMES
00288eac50a8   mysql:5.7   "docker-entrypoint.s…"   About a minute ago   Up 59 seconds   33060/tcp, 0.0.0.0:13306->3306/tcp   mysql-service

5. 進入容器操作
[demo@2t10 ~]$ sudo docker exec -it mysql-service /bin/sh
# mysql -uroot -p               # ---- > 登錄
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.7.35 MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> create database k3s default charset utf8mb4;     # ---- > 創(chuàng)建k3s庫
Query OK, 1 row affected (0.00 sec)

mysql> create user k3s@'10.2.2.10' identified by 'testdbk3s';     # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> create user k3s@'10.2.2.11' identified by 'testdbk3s';     # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> create user k3s@'10.2.2.12' identified by 'testdbk3s';    # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> create user k3s@'10.2.2.13' identified by 'testdbk3s';    # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> create user k3s@'10.2.2.14' identified by 'testdbk3s';    # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> create user k3s@'10.2.2.15' identified by 'testdbk3s';    # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> create user k3s@'10.2.2.16' identified by 'testdbk3s';    # ---- > 創(chuàng)建集群用戶并設(shè)置密碼
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.10';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.01 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.11';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.12';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.13';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.14';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.15';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on k3s.* to k3s@'10.2.2.16';    # ---- > 授權(quán)
Query OK, 0 rows affected (0.00 sec)

mysql> flush privileges;      # ---- > 刷新權(quán)限
Query OK, 0 rows affected (0.00 sec

六舟陆、安裝k3s server節(jié)點

對于server-1 10.2.2.13節(jié)點

1. 下載k3s離線安裝文件
[demo@2t13 k3s]$ pwd
/home/demo/k3s
[demo@2t13 k3s]$ ls -l
total 462584
-rw-rw-r-- 1 demo demo     26929 Oct 16 00:57 install.sh
-rw-rw-r-- 1 demo demo  56553472 Oct 16 00:57 k3s
-rw-rw-r-- 1 demo demo 417101824 Oct 16 00:57 k3s-airgap-images-amd64.tar
說明（下面是本文檔用到的程序版本）： 
install.sh 腳本內(nèi)容地址：https://get.k3s.io/
k3s 是k3s主程序。下載地址：https://github.com/k3s-io/k3s/releases/tag/v1.19.15+k3s2
k3s-airgap-images-amd64.tar 是k3s用到的鏡像耻矮。  下載地址：https://github.com/k3s-io/k3s/releases/tag/v1.19.15+k3s2

2. docker導入k3s-airgap-images-amd64.tar鏡像
[demo@2t13 k3s]$ sudo docker load -i k3s-airgap-images-amd64.tar

3. 給k3s執(zhí)行權(quán)限并復制到指定目錄
[demo@2t13 k3s]$ chmod +x k3s && sudo cp k3s /usr/local/bin/

4. 執(zhí)行安裝
# 將下面兩行加入到k3s的安裝腳本中,加到最上面
[demo@2t13 k3s]$ vim install.sh
... ...
export INSTALL_K3S_SKIP_DOWNLOAD=true
export INSTALL_K3S_EXEC="server --datastore-endpoint=mysql://k3s:testdbk3s@tcp(10.2.2.10:13306)/k3s --docker --node-taint CriticalAddonsOnly=true:NoExecute --tls-san 10.2.2.100 --kube-apiserver-arg service-node-port-range=10000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
... ...

注意： --tls-san參數(shù)后面的地址是為集群提供SLB的地址秦躯，對應為下文中keepalived的虛擬VIP地址

# 執(zhí)行腳本
[demo@2t13 k3s]$ sudo ./install.sh 
[INFO]  Skipping k3s download and verify
[INFO]  Skipping installation of SELinux RPM
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Skipping /usr/local/bin/ctr symlink to k3s, command exists in PATH at /bin/ctr
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]  systemd: Enabling k3s unit
Created symlink from /etc/systemd/system/multi-user.target.wants/k3s.service to /etc/systemd/system/k3s.service.
[INFO]  systemd: Starting k3s

5. 從root用戶家目錄拷貝kubeconfig文件到當前用戶家目錄
[demo@2t13 k3s]$ sudo cp -ar /root/.kube/config /home/demo/

6. 查看節(jié)點
[demo@2t13 k3s]$ kubectl get node
NAME                STATUS   ROLES    AGE     VERSION
2t13.cluster.ydca   Ready    master   2m48s   v1.19.15+k3s2

7. 查看token
[demo@2t13 k3s]$ sudo cat /var/lib/rancher/k3s/server/node-token
K10e1b1fcb4caf1f726580e0fb22d15ff4fcb48e5a26c0841b4c63b8176169a66f2::server:447912a715c422f1cce5893c37572280

對于server-2 10.2.2.14節(jié)點

該節(jié)點與server-1操作只有1處不同，如下
在server-1的第4步驟中裆装，我們向install.sh 腳本中添加了兩個環(huán)境變量
該變量在server-2節(jié)點應該改為如下：
export INSTALL_K3S_SKIP_DOWNLOAD=true
export INSTALL_K3S_EXEC="server --token K10e1b1fcb4caf1f726580e0fb22d15ff4fcb48e5a26c0841b4c63b8176169a66f2::server:447912a715c422f1cce5893c37572280 --datastore-endpoint=mysql://k3s:testdbk3s@tcp(10.2.2.10:13306)/k3s --docker --node-taint CriticalAddonsOnly=true:NoExecute --tls-san 10.2.2.100 --kube-apiserver-arg service-node-port-range=10000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"

說明：相比server-1中踱承，只是增加了--token 選項,該選項的數(shù)據(jù)來自server-1中的第7步驟。

兩個server節(jié)點部署完成后哨免，查看當前集群節(jié)點狀況

[demo@2t13 k3s]$ kubectl get node
NAME                STATUS   ROLES    AGE   VERSION
2t13.cluster.ydca   Ready    master   18m   v1.19.15+k3s2
2t14.cluster.ydca   Ready    master   12s   v1.19.15+k3s2

七茎活、使用 haproxy+keepalived 實現(xiàn) server 節(jié)點負載均衡及高可用

1、haproxy的部署和配置
部署 haproxy-2.4.7
部署方式：二進制
執(zhí)行范圍：10.2.2.11 10.2.2.12 （兩臺服務器操作完全一致）

1. 下載二進制包
下載地址：http://www.haproxy.org/

2. 安裝gcc
[demo@2t11 haproxy]$ sudo yum install gcc -y

2. 上傳二進制包到服務器并解壓
[demo@2t11 haproxy]$ tar -xvf haproxy-2.4.7.tar.gz  #------------------->解壓
[demo@2t11 haproxy]$ ls -lth  #------------------->查看
total 3.5M
-rw-rw-r--  1 demo demo 3.5M Oct 16 21:37 haproxy-2.4.7.tar.gz
drwxrwxr-x 13 demo demo 4.0K Oct  4 20:56 haproxy-2.4.7

3. 查看系統(tǒng)參數(shù)
[demo@2t11 haproxy]$ uname -a
Linux 2t11.cluster 4.4.246-1.el7.elrepo.x86_64 #1 SMP Tue Nov 24 09:26:59 EST 2020 x86_64 x86_64 x86_64 GNU/Linux

4. 安裝
[demo@2t11 haproxy]$ cd haproxy-2.4.7  #------------------->進入解壓后目錄
[demo@2t11 haproxy-2.4.7]$ sudo make TARGET=linux-glibc ARCH=x86_64 PREFIX=/usr/local/haproxy  #------------------->編譯
[demo@2t11 haproxy-2.4.7]$ sudo make install PREFIX=/usr/local/haproxy  #------------------->安裝

5. haproxy配置文件
[demo@2t11 haproxy]$ sudo mkdir /usr/local/haproxy/cfg #------------------->創(chuàng)建配置文件目錄
[demo@2t11 haproxy]$ cat /usr/local/haproxy/cfg/haproxy.cfg #------------------->配置文件內(nèi)容
global
  daemon
  maxconn 4000
  pidfile /usr/local/haproxy/haproxy.pid

defaults
  log global
  option  httplog
  option  dontlognull
  timeout connect 5000
  timeout client 50000
  timeout server 50000

listen admin_stats  #------------>開啟監(jiān)控頁面UI配置段
  stats   enable
  bind    *:18090
  mode    http
  option  httplog
  log     global
  maxconn 10
  stats   refresh 5s
  stats   uri /admin #------------>訪問URI配置
  stats   realm haproxy
  stats   auth admin:HaproxyProd1212!@2021 #------------>登錄名及密碼配置
  stats   hide-version
  stats   admin if TRUE

frontend k3s-apiserver #------------>定義代理入口配置段
  bind *:6443 #------------>定義代理端口
  mode tcp
  option tcplog
  default_backend k3s-apiserver #------------>定義代理后端

backend k3s-apiserver #------------>定義代理后端配置段
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100 #------------>定義負載均衡規(guī)則
  server k3s-apiserver-13 10.2.2.13:6443 check #------------>定義負載目標
  server k3s-apiserver-14 10.2.2.14:6443 check #------------>定義負載目標

6. 啟動腳本（注意腳本要有執(zhí)行權(quán)限）
[demo@2t11 haproxy]$ cat /usr/lib/systemd/system/haproxy.service  #------------------->啟動腳本內(nèi)容
[Unit]
Description=HAProxy
After=network.target

[Service]
User=root
Type=forking
ExecStart=/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/cfg/haproxy.cfg
ExecStop=/usr/bin/kill `/usr/bin/cat /usr/local/haproxy/haproxy.pid`

[Install]
WantedBy=multi-user.target

7. 啟動haproxy
[demo@2t12 haproxy-2.4.7]$ sudo systemctl daemon-reload
[demo@2t12 haproxy-2.4.7]$ sudo systemctl start haproxy
[demo@2t12 haproxy-2.4.7]$ sudo systemctl enable haproxy

8. 查看
[demo@2t11 haproxy]$ sudo netstat -tnlp|grep haproxy
tcp        0      0 0.0.0.0:18090           0.0.0.0:*               LISTEN      9340/haproxy        
tcp        0      0 0.0.0.0:6443            0.0.0.0:*               LISTEN      9340/haproxy        

訪問haproxy-UI http://10.2.2.11:18090/admin 監(jiān)控頁面

監(jiān)控頁面

2琢唾、 keepalived的部署和配置
部署 keepalived-2.1.5
部署方式：二進制
執(zhí)行范圍：10.2.2.11 10.2.2.12 （兩臺服務器keepalived配置文件有所差異载荔，下文會標明）

1. 下載二進制包
https://www.keepalived.org/download.html

2. 上傳到服務器并解壓
[demo@2t11 keepalived]$ tar -xvf keepalived-2.1.5.tar.gz

3. 安裝依賴
[demo@2t11 keepalived]$ sudo yum install curl gcc openssl-devel libnl3-devel net-snmp-devel -y

4. 配置
 [demo@2t11 keepalived]$ cd keepalived-2.1.5 #------------------->進入解壓后目錄
 [demo@2t11 keepalived]$ sudo ./configure --prefix=/usr/local/keepalived/ --sysconfdir=/etc #------------------->配置

5. 編譯、安裝
 [demo@2t11 keepalived]$ sudo make && sudo make install

6. 查看安裝目錄,并將相關(guān)文件復制到指定位置
[demo@2t11 keepalived-2.1.5]$ ls /usr/local/keepalived/
bin  etc  sbin  share
[demo@2t11 keepalived-2.1.5]$ pwd   #------------------->當前所在目錄為源碼包解壓后的目錄（編譯安裝時的目錄）
/home/demo/keepalived/keepalived-2.1.5
[demo@2t11 keepalived-2.1.5]$ sudo cp /usr/local/keepalived/sbin/keepalived /usr/sbin/
[demo@2t11 keepalived-2.1.5]$ sudo cp /usr/local/keepalived/bin/genhash /usr/sbin/
[demo@2t11 keepalived-2.1.5]$ sudo cp keepalived/keepalived.service /usr/lib/systemd/system/
[demo@2t11 keepalived-2.1.5]$ sudo cp keepalived/etc/init.d/keepalived.rh.init /etc/sysconfig/keepalived.sysconfig

7. 編寫配置文件
# 對于10.2.2.11
[demo@2t11 keepalived]$ cd /etc/keepalived #------------>進入配置文件目錄
[demo@2t11 keepalived]$ sudo mv keepalived.conf keepalived.conf.bak #------------>備份默認配置文件,使用下面的配置
[demo@2t11 keepalived]$ cat keepalived.conf #------------>配置文件
global_defs {
   notification_email {
       mail@lizhip.cn # 指定keepalived在發(fā)生切換時需要發(fā)送email到的對象采桃，一行一個
   }
   notification_email_from 2691905373@qq.com # 指定發(fā)件人 
   smtp_server smtp.qq.com  # smtp 服務器地址
   smtp_connect_timeout 30  # smtp 服務器連接超時時間
   router_id 2t11.cluster  # 標識本節(jié)點的字符串,通常為hostname,但不一定非得是hostname,故障發(fā)生時,郵件通知會用到
   script_user root
   enable_script_security
}

vrrp_script check_haproxy {
   script /etc/keepalived/check_haproxy.sh # haproxy狀態(tài)監(jiān)控腳本
   interval 3
} 

vrrp_instance VI_1 {
    state BACKUP # 節(jié)點角色懒熙，下面配置了不爭搶模式，故設(shè)置為BACKUP
    nopreempt # 不爭搶模式
    interface ens33 # 節(jié)點固有IP（非VIP）的網(wǎng)卡普办，用來發(fā)VRRP包做心跳檢測
    virtual_router_id 62 # 虛擬路由ID,取值在0-255之間,用來區(qū)分多個instance的VRRP組播,同一網(wǎng)段內(nèi)ID不能重復;主備必須為一樣;
    priority 100 # 用來選舉master的,要成為master那么這個選項的值最好高于其他機器50個點,該項取值范圍是1-255(在此范圍之外會被識別成默認值100)
    advert_int 1 # 檢查間隔默認為1秒,即1秒進行一次master選舉(可以認為是健康查檢時間間隔)
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.2.2.100 # 虛擬VIP地址,允許多個
    }
    track_script {
        check_haproxy
    }
}

# 對于10.2.2.12
global_defs {
   notification_email {
       mail@lizhip.cn
   }
   notification_email_from 2691905373@qq.com
   smtp_server smtp.qq.com
   smtp_connect_timeout 30
   router_id 2t11.cluster
   script_user root
   enable_script_security
}

vrrp_script check_haproxy {
   script /etc/keepalived/check_haproxy.sh
   interval 3
} 

vrrp_instance VI_1 {
    state BACKUP
    nopreempt
    interface ens33
    virtual_router_id 62
    priority 99  # -----------------------> 此處與10.2.2.11不一樣
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.2.2.100
    }
    track_script {
        check_haproxy
    }
}

8. check_haproxy腳本（注意腳本要有執(zhí)行權(quán)限）
[demo@2t12 keepalived-2.1.5]$ cat /etc/keepalived/check_haproxy.sh
#!/bin/bash
haproxy_status=`/usr/sbin/pidof haproxy|wc -l`
if [ $haproxy_status -lt 1 ];then
     systemctl stop keepalived
fi

9. 啟停管理
[demo@2t12 keepalived-2.1.5]$ sudo systemctl daemon-reload
[demo@2t12 keepalived-2.1.5]$ sudo systemctl start keepalived
[demo@2t12 keepalived-2.1.5]$ sudo systemctl stop keepalived
[demo@2t12 keepalived-2.1.5]$ sudo systemctl enable keepalived

3工扎、通過keepalived虛擬VIP http://10.2.2.100:18090/admin 訪問haproxy監(jiān)控頁面

image.png

八畴博、安裝k3s agent節(jié)點

執(zhí)行范圍： 10.2.2.15 10.2.2.16
登錄10.2.2.13（即上述server-1節(jié)點）笨忌，拷貝k3s的3個文件到10.2.2.15和10.2.2.16兩臺主機上

[demo@2t13 k3s]$ cd /home/demo/k3s/ # ---> 進入k3s文件目錄
[demo@2t13 k3s]$ ls # ---> 查看
install.sh  k3s  k3s-airgap-images-amd64.tar # ---> 就是這3個文件
[demo@2t13 k3s]$ scp ./* 10.2.2.15:/home/demo/k3s/ # ---> 拷貝到10.2.2.15
[demo@2t13 k3s]$ scp ./* 10.2.2.16:/home/demo/k3s/ # ---> 拷貝到10.2.2.16

修改install.sh文件蓝仲，如下（10.2.2.15和10.2.2.16改動都一樣）

[demo@2t15 k3s]$  vim install.sh
... ...
export INSTALL_K3S_SKIP_DOWNLOAD=true
export K3S_TOKEN=K10e1b1fcb4caf1f726580e0fb22d15ff4fcb48e5a26c0841b4c63b8176169a66f2::server:447912a715c422f1cce5893c37572280
export K3S_URL=https://10.2.2.100:6443
export INSTALL_K3S_EXEC="agent --datastore-endpoint=mysql://k3s:testdbk3s@tcp(10.2.2.10:13306)/k3s --docker --kube-apiserver-arg service-node-port-range=10000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
... ...

安裝

1. 給k3s執(zhí)行權(quán)限并復制到指定目錄
[demo@2t15 k3s]$ chmod +x k3s && sudo cp k3s /usr/local/bin/

2. 執(zhí)行
[demo@2t15 k3s]$ sudo ./install.sh 
[sudo] password for demo: 
[INFO]  Skipping k3s download and verify
[INFO]  Skipping installation of SELinux RPM
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Skipping /usr/local/bin/ctr symlink to k3s, command exists in PATH at /bin/ctr
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-agent-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s-agent.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s-agent.service
[INFO]  systemd: Enabling k3s-agent unit
Created symlink from /etc/systemd/system/multi-user.target.wants/k3s-agent.service to /etc/systemd/system/k3s-agent.service.
[INFO]  systemd: Starting k3s-agent

登錄10.2.2.13查看集群節(jié)點

[demo@2t13 k3s]$ kubectl get node
NAME                STATUS   ROLES    AGE   VERSION
2t16.cluster   Ready    <none>   15m   v1.19.15+k3s2
2t14.cluster   Ready    master   33m   v1.19.15+k3s2
2t13.cluster   Ready    master   77m   v1.19.15+k3s2
2t15.cluster   Ready    <none>   16m   v1.19.15+k3s2

九、安裝rancher-ui界面
操作范圍：10.2.2.10

[demo@2t10 ~]$ sudo docker run --privileged -d -v /home/demo/rancherUI-data/:/var/lib/rancher --restart=unless-stopped --name rancher -p 80:80 -p 9443:443 rancher/rancher:v2.4.17
c93d4d3f1a273cb693d6caf3f515d88797172a81f36a3acf5ce2f75138e46e9e

訪問

image.png

image.png

image.png

image.png

繼續(xù)按下圖所示導入k3s集群到rancher

image.png

image.png

image.png

image.png

image.png

image.png

image.png