haproxy+keepalived 集群高可用集群轉(zhuǎn)發(fā)

環(huán)境介紹

#內(nèi)核版本
Ubuntu 18.04.4 LTS \n \l
107-Ubuntu SMP Thu Jun 4 11:27:52 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
#節(jié)點(diǎn)介紹
192.168.1.113  hk-master1
192.168.1.114  hk-master2
192.168.1.111  hk-slave1
192.168.1.112  hk-slave2

內(nèi)核調(diào)優(yōu)

#調(diào)整Linux進(jìn)程資源限制 vim /etc/security/limits.conf 
root soft core unlimited
root hard core unlimited
root soft nproc 600000
root hard nproc 600000
root soft nofile 648576
root hard nofile 600000
root soft memlock 32000
root hard memlock 32000
root soft msgqueue 8192000
root hard msgqueue 8192000

* soft core unlimited
* hard core unlimited
* soft nproc 600000
* hard nproc 600000
* soft nofile 600000
* hard nofile 600000
* soft memlock 32000
* hard memlock 32000
* soft msgqueue 8192000
* hard msgqueue 8192000
#驗(yàn)證（進(jìn)程對(duì)資源的使用情況）
root@hk-master2:~# ulimit -a
core file size          (blocks, -c) unlimited
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 7376
max locked memory       (kbytes, -l) 32000
max memory size         (kbytes, -m) unlimited
open files                      (-n) 600000
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 8192000
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 600000
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

#調(diào)整內(nèi)核限制追加以下配置 /etc/sysctl.conf
 net.ipv4.conf.default.rp_filter = 1
 net.ipv4.ip_nonlocal_bind = 1 
 net.ipv4.ip_forward = 1 
 net.ipv4.conf.default.accept_source_route = 0
 kernel.sysrq = 0
 kernel.msgmnb = 65536
 kernel.msgmax = 65536
 kernel.shmmax = 68719476736
 kernel.shmall = 4294967296
 net.ipv4.tcp_mem = 786432 1048576 1572864
 net.ipv4.tcp_rmem = 4096 87380 4194304
 net.ipv4.tcp_wmem = 4096 16384 4194304
 net.ipv4.tcp_window_scaling = 1
 net.ipv4.tcp_sack = 1
 net.core.wmem_default = 8388608
 net.core.rmem_default = 8388608
 net.core.rmem_max = 16777216
 net.core.wmem_max = 16777216
 net.core.netdev_max_backlog = 262144
 net.core.somaxconn = 20480
 net.core.optmem_max = 81920
 net.ipv4.tcp_max_syn_backlog = 262144
 net.ipv4.tcp_syn_retries = 3
 net.ipv4.tcp_retries1 = 3
 net.ipv4.tcp_retries2 = 15
 net.ipv4.tcp_timestamps = 0 #代理不要開(kāi)這個(gè)
 net.ipv4.tcp_fin_timeout = 1
 net.ipv4.tcp_max_tw_buckets = 20000
 net.ipv4.tcp_max_orphans = 3276800
 net.ipv4.tcp_synack_retries = 1
 net.ipv4.tcp_syncookies = 1
 net.ipv4.tcp_keepalive_time = 300
 net.ipv4.tcp_keepalive_intvl = 30
 net.ipv4.tcp_keepalive_probes = 3
 net.ipv4.ip_local_port_range = 10001 65000
 vm.overcommit_memory = 0
 vm.swappiness = 10
#驗(yàn)證 sysctl -p

haproxy安裝和功能介紹

#安裝
root@hk-master2:~# apt install -y haproxy
root@hk-master1:~# apt install -y haproxy

配置介紹

配置文件目錄

主程序：/usr/sbin/haproxy
配置文件：/etc/haproxy/haproxy.cfg
Unit file：/usr/lib/systemd/system/haproxy.service

配置段：

#global 配置:
? chroot #鎖定運(yùn)行目錄
? deamon #以守護(hù)進(jìn)程運(yùn)行
? #stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin #socket文件
? user, group, uid, gid #運(yùn)行haproxy的用戶身份
? nbproc #開(kāi)啟的haproxy進(jìn)程數(shù)，與CPU保持一致
? nbthread #指定每個(gè)haproxy進(jìn)程開(kāi)啟的線程數(shù)，默認(rèn)為每個(gè)進(jìn)程一個(gè)線程
? cpu-map 1 0 #綁定haproxy 進(jìn)程至指定CPU
? maxconn #每個(gè)haproxy進(jìn)程的最大并發(fā)連接數(shù)
? maxsslconn #SSL每個(gè)haproxy進(jìn)程ssl最大連接數(shù)
? maxconnrate #每個(gè)進(jìn)程每秒最大連接數(shù)
? spread-checks #后端server狀態(tài)check隨機(jī)提前或延遲百分比時(shí)間肖揣，建議2-5(20%-50%)之間
? pidfile #指定pid文件路徑
? log 127.0.0.1 local3 info #定義全局的syslog服務(wù)器齿梁；最多可以定義兩個(gè)
? defaults [<name>] #默認(rèn)配置項(xiàng)悯舟，針對(duì)以下的frontend患整、backend和lsiten生效，可以多個(gè)name
? frontend <name> #前端servername济舆，類似于Nginx的一個(gè)虛擬主機(jī) server卿泽。 ? backend <name> #后端服務(wù)器組，等于nginx的upstream
? listen <name> #將frontend和backend合并在一起配置
? 注：name字段只能使用”-”滋觉、”_”签夭、”.”、和”:”椎侠，并且嚴(yán)格區(qū)分大小寫第租，例如：Web和web是完全不
同的兩組服務(wù)器。


#defaults 配置參數(shù):
? option redispatch #當(dāng)server Id對(duì)應(yīng)的服務(wù)器掛掉后我纪，強(qiáng)制定向到其他健康的服務(wù)器
? option abortonclose #當(dāng)服務(wù)器負(fù)載很高的時(shí)候慎宾，自動(dòng)結(jié)束掉當(dāng)前隊(duì)列處理比較久的鏈接
? option http-keep-alive 60#開(kāi)啟會(huì)話保持
? option forwardfor #開(kāi)啟IP透?jìng)?? mode http #默認(rèn)工作類型
? timeout connect 120s #轉(zhuǎn)發(fā)客戶端請(qǐng)求到后端server的最長(zhǎng)連接時(shí)間(TCP之前) ? timeout server 600s #轉(zhuǎn)發(fā)客戶端請(qǐng)求到后端服務(wù)端的超時(shí)超時(shí)時(shí)長(zhǎng)（TCP之后）
? timeout client 600s #與客戶端的最長(zhǎng)空閑時(shí)間
? timeout http-keep-alive 120s #session 會(huì)話保持超時(shí)時(shí)間，范圍內(nèi)會(huì)轉(zhuǎn)發(fā)到相同的后端服務(wù)器
? #timeout check 5s #對(duì)后端服務(wù)器的檢測(cè)超時(shí)時(shí)間

#listen 配置參考：
listen WEB_PORT_80
    bind 192.168.7.102:80
    mode http
    option forwardfor
    server web1 192.168.7.101:8080 check inter 3000 fall 3 rise 5
    server web2 192.168.7.101:8080 check inter 3000 fall 3 rise 5

#后端服務(wù)器檢測(cè)機(jī)制參數(shù)介紹：
check #對(duì)指定real進(jìn)行健康狀態(tài)檢查浅悉，默認(rèn)不開(kāi)啟
? addr IP #可指定的健康狀態(tài)監(jiān)測(cè)IP
? port num #指定的健康狀態(tài)監(jiān)測(cè)端口
? inter num #健康狀態(tài)檢查間隔時(shí)間趟据，默認(rèn)2000 ms
? fall num #后端服務(wù)器失效檢查次數(shù)，默認(rèn)為3 ? rise num #后端服務(wù)器從下線恢復(fù)檢查次數(shù)术健，默認(rèn)為2 ? weight #默認(rèn)為1汹碱，最大值為256，0表示不參與負(fù)載均衡
? backup #將后端服務(wù)器標(biāo)記為備份狀態(tài)
? disabled #將后端服務(wù)器標(biāo)記為不可用狀態(tài)
? redirect prefix http://www.magedu.com/ #將請(qǐng)求臨時(shí)重定向至其它URL荞估，只適用于http模式
? maxconn <maxconn>：當(dāng)前后端server的最大并發(fā)連接數(shù)
? backlog <backlog>：當(dāng)server的連接數(shù)達(dá)到上限后的后援隊(duì)列長(zhǎng)度

調(diào)度算法

靜態(tài)調(diào)度算法

balance： 指明對(duì)后端服務(wù)器的調(diào)度算法咳促，配置在listen或backend
靜態(tài)算法：按照事先定義好的規(guī)則輪詢公平調(diào)度，不關(guān)心后端服務(wù)器的當(dāng)前負(fù)載勘伺、鏈接數(shù)和相應(yīng)速度等跪腹，且無(wú)法實(shí)時(shí)修改權(quán)重，只能重啟后生效娇昙。
static-rr：基于權(quán)重的輪詢調(diào)度尺迂，不支持權(quán)重的運(yùn)行時(shí)調(diào)整及后端服務(wù)器慢啟動(dòng)，其后端主機(jī)數(shù)量沒(méi)有限制 (出現(xiàn)請(qǐng)求按比例分發(fā)給后端)
first：根據(jù)服務(wù)器在列表中的位置，自上而下進(jìn)行調(diào)度噪裕，但是其只會(huì)當(dāng)?shù)谝慌_(tái)服務(wù)器的連接數(shù)達(dá)到上限蹲盘，新請(qǐng)求才會(huì)分配給下一臺(tái)服務(wù)，因此會(huì)忽略服務(wù)器的權(quán)重設(shè)置膳音。  （配置的后端服務(wù)器連接數(shù)到了上線召衔，才會(huì)分發(fā)到下臺(tái)后端服務(wù)器）

動(dòng)態(tài)調(diào)度算法

動(dòng)態(tài)算法：基于后端服務(wù)器 狀態(tài)進(jìn)行調(diào)度適當(dāng)調(diào)整，比如優(yōu)先調(diào)度至當(dāng)前負(fù)載較低的服務(wù)器祭陷，且權(quán)重可以在haproxy運(yùn)行時(shí)動(dòng)態(tài)調(diào)整無(wú)需重啟苍凛。
roundrobin：基于權(quán)重的輪詢動(dòng)態(tài)調(diào)度算法，支持權(quán)重的運(yùn)行時(shí)調(diào)整兵志，不等于lvs 的rr醇蝴，支持慢啟動(dòng)即新加的服務(wù)器會(huì)逐漸增加轉(zhuǎn)發(fā)數(shù)，每個(gè)后端backend中最多支持4095個(gè)server想罕，此為默認(rèn)調(diào)度算法悠栓，server 權(quán)重設(shè)置 weight
leastconn： 加權(quán)的最少連接的動(dòng)態(tài)，支持權(quán)重的運(yùn)行時(shí)調(diào)整和慢啟動(dòng)按价，即當(dāng)前后端服務(wù)器連接最少的優(yōu)先調(diào)度惭适，比較適合長(zhǎng)連接的場(chǎng)景使用，比如MySQL等場(chǎng)景楼镐。

source調(diào)度算法

source：源地址hash癞志，基于用戶源地址hash并將請(qǐng)求轉(zhuǎn)發(fā)到后端服務(wù)器，默認(rèn)為靜態(tài)即取模方式框产，但是可以通過(guò)hash-type支持的選項(xiàng)更改凄杯，后續(xù)同一個(gè)源地址請(qǐng)求將被轉(zhuǎn)發(fā)至同一個(gè)后端web服務(wù)器，比較適用于session保持/緩存業(yè)務(wù)等場(chǎng)景茅信。

? map-based：取模法盾舌，基于服務(wù)器權(quán)重的hash數(shù)組取模，該hash是靜態(tài)的即不支持在線調(diào)整權(quán)重蘸鲸，不支持慢啟動(dòng)，其對(duì)后端服務(wù)器調(diào)度均衡窿锉，缺點(diǎn)是當(dāng)服務(wù)器的總權(quán)重發(fā)生變化時(shí)酌摇，即有服務(wù)器上線或下線，都會(huì)因權(quán)重發(fā)生變化而導(dǎo)致調(diào)度結(jié)果整體改變hash（o）mod n 嗡载。

?consistent：一致性哈希窑多，該hash是動(dòng)態(tài)的，支持在線調(diào)整權(quán)重洼滚，支持慢啟動(dòng)埂息，優(yōu)點(diǎn)在于當(dāng)服務(wù)器的總權(quán)重發(fā)生變化時(shí)，對(duì)調(diào)度結(jié)果影響是局部的，不會(huì)引起大的變動(dòng)千康。

#配置案例：
listen web_prot_http_nodes
    bind 192.168.7.101:80
    mode http
    balance source
    hash-type consistent
    log global
    option forwardfor
    server 192.168.7.101 192.168.7.101:8080 check inter 3000 fall 3 rise 5
    server 192.168.7.102 192.168.7.102:8080 check inter 3000 fall 3 rise 5

uri調(diào)度算法

uri：基于對(duì)用戶請(qǐng)求的uri做hash并將請(qǐng)求轉(zhuǎn)發(fā)到后端指定服務(wù)器
? map-based：取模法
? consistent：一致性哈希

listen web_prot_http_nodes
    bind 192.168.7.101:80
    mode http #不支持tcp享幽，會(huì)切換到tcp的roundrobin負(fù)載模式
    balance uri
    hash-type consistent
    log global
    option forwardfor
    server 192.168.7.101 192.168.7.101:8080 check inter 3000 fall 3 rise 5
    server 192.168.7.102 192.168.7.102:8080 check inter 3000 fall 3 rise 5

url_param 調(diào)度算法

#url_param： 對(duì)用戶請(qǐng)求的url中的<params>部分中的參數(shù)name作hash計(jì)算，并由服務(wù)器總權(quán)重相除以后派發(fā)至某挑出的服務(wù)器拾弃；通常用于追蹤用戶值桩，以確保來(lái)自同一個(gè)用戶的請(qǐng)求始終發(fā)往同一個(gè)Backend Server
#url 傳遞的查詢字符串進(jìn)行bash
listen web_prot_http_nodes
    bind 192.168.7.101:80
    mode http #不支持tcp，會(huì)切換到tcp的roundrobin負(fù)載模式
    balance url_param name #基于參數(shù)name做hash
    hash-type consistent
    log global
    option forwardfor
    server 192.168.7.101 192.168.7.101:8080 check inter 3000 fall 3 rise 5
    server 192.168.7.102 192.168.7.102:8080 check inter 3000 fall 3 rise 5

hdr調(diào)度算法

#針對(duì)每個(gè)用戶的http請(qǐng)求頭中的指定信息做hash,此處由<name>指定的http首部將會(huì)被取出并做hash計(jì)算豪椿，然后由服務(wù)器總權(quán)重相除以后派發(fā)至某挑出的服務(wù)器奔坟，假如無(wú)有效的值，則會(huì)被輪詢調(diào)度
hdr( Cookie搭盾、 User-Agent咳秉、host )

listen web_prot_http_nodes
    bind 192.168.7.101:80
    mode http
    balance hdr(User-Agent)
    hash-type consistent #一致性hash 
    log global
    option forwardfor
    server 192.168.7.101 192.168.7.101:8080 check inter 3000 fall 3 rise 5
    server 192.168.7.102 192.168.7.102:8080 check inter 3000 fall 3 rise 5

rdp-cookie調(diào)度算法

rdp-cookie對(duì)遠(yuǎn)程桌面的負(fù)載，使用cookie保持會(huì)話

listen RDP
    bind 192.168.7.101:3389
    balance rdp-cookie
    mode tcp
    server rdp0 172.18.139.20:3389 check fall 3 rise 5 inter 2000 weight 1
    server rdp1 172.18.139.21:3389 check fall 3 rise 5 inter 2000 weight 1
    
    

配置狀態(tài)頁(yè)

stats enable #基于默認(rèn)的參數(shù)啟用stats page
stats hide-version # 隱藏版本
stats refresh <delay> # 設(shè)定自動(dòng)刷新時(shí)間間隔
stats uri <prefix> #自定義stats page uri鸯隅，默認(rèn)值：/haproxy?stats 
stats realm <realm> #賬戶認(rèn)證時(shí)的提示信息澜建，示例：stats realm : HAProxy\ Statistics
stats auth <user>:<passwd> #認(rèn)證時(shí)的賬號(hào)和密碼，可使用多次滋迈，默認(rèn)：no authentication
stats admin { if | unless } <cond> #啟用stats page中的管理功能


listen stats
    bind :9009
    stats enable
    #stats hide-version 
    stats uri /haproxy-status
    stats realm HAPorxy\ Stats\ Page
    stats auth haadmin:123456
    stats auth admin:123456
    stats refresh 30s
    stats admin if TRUE

自定義錯(cuò)誤頁(yè)面

errorfile 500 /usr/local/haproxy/html/500.html #自定義錯(cuò)誤頁(yè)面跳轉(zhuǎn)
errorfile 502 /usr/local/haproxy/html/502.html
errorfile 503 /usr/local/haproxy/html/503.html

errorloc 503 http://192.168.7.103/error_page/503.html

壓縮功能

compression algo #啟用http協(xié)議中的壓縮機(jī)制霎奢，常用算法有g(shù)zip deflate
compression type #要壓縮的類型
? 示例：
    ? compression algo gzip
    ? compression type compression type text/plain text/html text/css text/xml text/javascript application/javascript

配置https

bind *:443 ssl crt /PATH/TO/SOME_PEM_FILE
    crt 后證書(shū)文件為PEM格式，且同時(shí)包含證書(shū)和所有私鑰
    cat demo.crt demo.key > demo.pem 
把80端口的請(qǐng)求重向定443
    bind *:80
    redirect scheme https if !{ ssl_fc }
向后端傳遞用戶請(qǐng)求的協(xié)議和端口（frontend或backend）
    http_request set-header X-Forwarded-Port %[dst_port]
    http_request add-header X-Forwared-Proto https if { ssl_fc }
#配置示例：
frontend https_frontend
  bind *:443 ssl crt /etc/ssl/certs/servername.pem
  mode http
  option httpclose
  option forwardfor
  reqadd X-Forwarded-Proto:\ https
  default_backend web_server

backend web_server
  mode http
  balance roundrobin
  cookie SERVERID insert indirect nocache
  server s1 192.168.250.47:80 check cookie s1
  server s2 192.168.250.49:80 check cookie s2
 注意：這里的pem 文件是下面兩個(gè)文件合并而成：
  cat servername.crt servername.key |tee servername.pem
  
#第二種四層轉(zhuǎn)發(fā)
frontend https_frontend
  bind *:443
  mode tcp
  default_backend web_server

backend web_server
  mode tcp
  balance roundrobin
  stick-table type ip size 200k expire 30m
  stick on src
  server s1 192.168.250.47:443
  server s2 192.168.250.49:443
  
  注意饼灿，這種模式下mode 必須是tcp 模式
         

四層負(fù)載IP透?jìng)?/h3>

#在四層負(fù)載設(shè)備中幕侠，把client發(fā)送的報(bào)文目標(biāo)地址(原來(lái)是負(fù)載均衡設(shè)備的IP地址)，根據(jù)均衡設(shè)備設(shè)置的選擇web服務(wù)器的規(guī)則選擇對(duì)應(yīng)的web服務(wù)器IP地址碍彭，這樣client就可以直接跟此服務(wù)器建立TCP連接并發(fā)送數(shù)據(jù)晤硕。

listen web_prot_http_nodes
    bind 192.168.7.102:80
    mode tcp
    server 192.168.7.102 blogs.studylinux.net:80 send-proxy check inter 3000 fall 3 rise 5 #send-proxy

Nginx配置：
    listen 80 proxy_protocol; 
        '"tcp_ip":"$proxy_protocol_addr",' #TCP獲取客戶端真實(shí)IP日志格式

七層負(fù)載IP透?jìng)?/h3>

#七層負(fù)載均衡服務(wù)器起了一個(gè)代理服務(wù)器的作用，服務(wù)器建立一次TCP連接要三次握手庇忌，而client要訪問(wèn)webserver要先與七層負(fù)載設(shè)備進(jìn)行三次握手后建立TCP連接舞箍，把要訪問(wèn)的報(bào)文信息發(fā)送給七層負(fù)載均衡；然后七層負(fù)載均衡再根據(jù)設(shè)置的均衡規(guī)則選擇特定的webserver皆疹，然后通過(guò)三次握手與此臺(tái)webserver建立TCP連接疏橄，然后webserver把需要的數(shù)據(jù)發(fā)送給七層負(fù)載均衡設(shè)備，負(fù)載均衡設(shè)備再把數(shù)據(jù)發(fā)送給client略就；所以捎迫，七層負(fù)載均衡設(shè)備起到了代理服務(wù)器的作用。

listen web_prot_http_nodes
    bind 192.168.7.102:80
    mode http
    #option forwardfor
    server 192.168.7.102 blogs.studylinux.net:80 check inter 3000 fall 3 rise 5

heepalived 安裝和功能介紹

root@hk-master2:~# apt install -y haproxy
root@hk-master1:~# apt install -y haproxy

功能

基于vrrp協(xié)議完成地址流動(dòng)
為vip地址所在的節(jié)點(diǎn)生成ipvs規(guī)則(在配置文件中預(yù)先定義) 
為ipvs集群的各RS做健康狀態(tài)檢測(cè)
基于腳本調(diào)用接口通過(guò)執(zhí)行腳本完成腳本中定義的功能表牢，進(jìn)而影響集群事務(wù)窄绒，以此支持nginx、haproxy等服務(wù)

環(huán)境要求

#個(gè)節(jié)點(diǎn)時(shí)間同步
#關(guān)閉selinux
#添加防火墻策略/關(guān)閉防火墻
firewall-cmd --direct --permanent --add-rule ipv4 filter INPUT 0 \
  --in-interface ens0 --destination 192.168.1.114 --protocol vrrp -j ACCEPT
success
firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT 0 \
  --out-interface ens0 --destination 192.168.1.114 --protocol vrrp -j ACCEPT
success
firewall-cmd --reload
success

配置介紹

主配置文件：/etc/keepalived/keepalived.conf
主程序文件：/usr/sbin/keepalived
Unit File： 
    ? /usr/lib/systemd/system/keepalived.service (CentOS) 
    ? /lib/systemd/system/keepalived.service (Ubuntu)

#配置文件組成部分
TOP HIERACHY
    GLOBAL CONFIGURATION
        Global definitions
        
    VRRP CONFIGURATION
        VRRP instance(s)：即一個(gè)vrrp虛擬路由器
        
    LVS CONFIGURATION
        Virtual server group(s)
        Virtual server(s)：ipvs集群的vs和rs
        
#配置參數(shù)：
state MASTER|BACKUP：當(dāng)前節(jié)點(diǎn)在此虛擬路由器上的初始狀態(tài)崔兴，狀態(tài)為MASTER或者BACKUP
interface IFACE_NAME：綁定為當(dāng)前虛擬路由器使用的物理接口ens32,ens0,bond0,br0
virtual_router_id VRID：當(dāng)前虛擬路由器惟一標(biāo)識(shí)彰导，范圍是0-255
priority 100：當(dāng)前物理節(jié)點(diǎn)在此虛擬路由器中的優(yōu)先級(jí)蛔翅；范圍1-254
advert_int 1：vrrp通告的時(shí)間間隔，默認(rèn)1s
authentication { #認(rèn)證機(jī)制
auth_type AH|PASS
auth_pass <PASSWORD> 僅前8位有效
}
virtual_ipaddress { #虛擬IP
    <IPADDR>/<MASK> brd <IPADDR> dev <STRING> scope <SCOPE> label <LABEL>
    192.168.200.17/24 dev ens1
    192.168.200.18/24 dev ens2 label ens2:1
}
track_interface { #配置監(jiān)控網(wǎng)絡(luò)接口位谋，一旦出現(xiàn)故障山析，則轉(zhuǎn)為FAULT狀態(tài)實(shí)現(xiàn)地址轉(zhuǎn)移
    ens0
    ens1
    … 
    }

組播配置

#master :
global_defs { 
    notification_email { 
        root@localhost #keepalived 發(fā)生故障切換時(shí)郵件發(fā)送的對(duì)象，可以按行區(qū)分寫多個(gè)
}
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id ha1.example.com
    vrrp_skip_check_adv_addr #所有報(bào)文都檢查比較消耗性能倔幼，此配置為如果收到的報(bào)文和上一個(gè)報(bào)文是同一個(gè)路由器則跳過(guò)檢查報(bào)文中的源地址
    vrrp_strict #嚴(yán)格遵守VRRP協(xié)議,不允許狀況:1,沒(méi)有VIP地址,2.單播鄰居,3.在VRRP版本2中有IPv6地 址. ? vrrp_garp_interval 0 #ARP報(bào)文發(fā)送延遲
    vrrp_gna_interval 0 #消息發(fā)送延遲
    vrrp_mcast_group4 224.0.0.18 #默認(rèn)組播IP地址盖腿，224.0.0.0到239.255.255.255
    #vrrp_iptables
    }
    vrrp_instance VI_1 {
        state MASTER
        interface ens0
        virtual_router_id 80
        priority 100
        advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111qwer
    } 
    virtual_ipaddress {
        192.168.7.248 dev ens0 label ens0:0
    } 
}
#backup :
global_defs {
    notification_email {
        root@localhost
} 
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id ha2.example.com
    vrrp_skip_check_adv_addr #
    vrrp_strict #嚴(yán)格遵守VRRP協(xié)議。
    vrrp_garp_interval 0 #ARP報(bào)文發(fā)送延遲
    vrrp_gna_interval 0 #消息發(fā)送延遲
    vrrp_mcast_group4 224.0.0.18 #組播IP地址损同，224.0.0.0到239.255.255.255
    #vrrp_iptables
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens0
    virtual_router_id 80
    priority 90
    advert_int 1
    authentication {
    auth_type PASS
    auth_pass 1111qwer
} 
virtual_ipaddress {
    192.168.7.248 dev ens0 label ens0:0
    } 
}

非搶占

#設(shè)置成雙備模式 關(guān)閉vip搶占 + nopreempt
#hk-master1
vrrp_instance VI_1 {
    state BACKUP
    interface ens0
    virtual_router_id 80
    priority 100
    advert_int 1 
    nopreempt
#hk-master2
vrrp_instance VI_1 {
    state BACKUP
    interface ens0
    virtual_router_id 80
    priority 90
    advert_int 1 
    nopreempt

單波配置

unicast_src_ip 本機(jī)源IP
    unicast_peer {
    目標(biāo)主機(jī)IP
}

通知配置

vim /etc/mail.rc
set from=12161xxqq.com
set smtp=smtp.qq.com
set smtp-auth-user=12161xxqq.com
set smtp-auth-password=xxxxxxx
set smtp-auth=login
set ssl-verify=ignore

nopreempt：定義工作模式為非搶占模式
preempt_delay 300：搶占式模式翩腐，節(jié)點(diǎn)上線后觸發(fā)新選舉操作的延遲時(shí)長(zhǎng)，
默認(rèn)模式
定義通知腳本：
    notify_master <STRING>|<QUOTED-STRING>：
        當(dāng)前節(jié)點(diǎn)成為主節(jié)點(diǎn)時(shí)觸發(fā)的腳本
    notify_backup <STRING>|<QUOTED-STRING>：
        當(dāng)前節(jié)點(diǎn)轉(zhuǎn)為備節(jié)點(diǎn)時(shí)觸發(fā)的腳本
    notify_fault <STRING>|<QUOTED-STRING>：
        當(dāng)前節(jié)點(diǎn)轉(zhuǎn)為“失敗”狀態(tài)時(shí)觸發(fā)的腳本
    notify <STRING>|<QUOTED-STRING>：
        通用格式的通知觸發(fā)機(jī)制膏燃，一個(gè)腳本可完成以上三種狀態(tài)的轉(zhuǎn)換時(shí)的通知

[圖片上傳失敗...(image-79b2d4-1593177183263)]

應(yīng)用層監(jiān)控

HTTP_GET|SSL_GET：應(yīng)用層檢測(cè)
HTTP_GET|SSL_GET {
    url {
        path <URL_PATH>：定義要監(jiān)控的URL
        status_code <INT>：判斷上述檢測(cè)機(jī)制為健康狀態(tài)的響應(yīng)碼
    }
connect_timeout <INTEGER>：連接請(qǐng)求的超時(shí)時(shí)長(zhǎng)
nb_get_retry <INT>：重試次數(shù)
delay_before_retry <INT>：重試之前的延遲時(shí)長(zhǎng)
connect_ip <IP ADDRESS>：向當(dāng)前RS哪個(gè)IP地址發(fā)起健康狀態(tài)檢測(cè)請(qǐng)求
connect_port <PORT>：向當(dāng)前RS的哪個(gè)PORT發(fā)起健康狀態(tài)檢測(cè)請(qǐng)求
bindto <IP ADDRESS>：發(fā)出健康狀態(tài)檢測(cè)請(qǐng)求時(shí)使用的源地址
bind_port <PORT>：發(fā)出健康狀態(tài)檢測(cè)請(qǐng)求時(shí)使用的源端口
}
#real_server http監(jiān)測(cè)
real_server 192.168.7.103 80 {
    weight 1
    HTTP_GET {
    url {
        path /index.html
        status_code 200
        } 
    }
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3 
}3

tcp監(jiān)控

傳輸層檢測(cè) TCP_CHECK
    TCP_CHECK {
        connect_ip <IP ADDRESS>：向當(dāng)前RS的哪個(gè)IP地址發(fā)起健康狀態(tài)檢測(cè)請(qǐng)求
        connect_port <PORT>：向當(dāng)前RS的哪個(gè)PORT發(fā)起健康狀態(tài)檢測(cè)請(qǐng)求
        bindto <IP ADDRESS>：發(fā)出健康狀態(tài)檢測(cè)請(qǐng)求時(shí)使用的源地址
        bind_port <PORT>：發(fā)出健康狀態(tài)檢測(cè)請(qǐng)求時(shí)使用的源端口
        connect_timeout <INTEGER>：連接請(qǐng)求的超時(shí)時(shí)長(zhǎng)
    }

腳本監(jiān)控

分兩步：(1) 先定義一個(gè)腳本茂卦；(2) 調(diào)用此腳本
vrrp_script <SCRIPT_NAME> {
    script <STRING>|<QUOTED-STRING>
    interval <INTEGER> # 間隔時(shí)間，單位為秒组哩，默認(rèn)1秒
    timeout <INTEGER> # 超時(shí)時(shí)間
    weight <INTEGER:-254..254> # 權(quán)重等龙，監(jiān)測(cè)失敗后會(huì)執(zhí)行權(quán)重+操作
    fall <INTEGER> #腳本幾次失敗轉(zhuǎn)換為失敗
    rise <INTEGER> # 腳本連續(xù)監(jiān)測(cè)成果后，把服務(wù)器從失敗標(biāo)記為成功的次數(shù)
    user USERNAME [GROUPNAME] # 執(zhí)行監(jiān)測(cè)的用戶或組
    init_fail # 設(shè)置默認(rèn)標(biāo)記為失敗狀態(tài)伶贰，監(jiān)測(cè)成功之后再轉(zhuǎn)換為成功狀態(tài)
}

vrrp_instance VI_1 {
…
track_script {
    SCRIPT_NAME_1
    SCRIPT_NAME_2
    }
}

配置案例：

#查找配置案例
root@hk-master2:~# find /usr/share/doc/keepalived/ -name keepalived.*
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.routes
/usr/share/doc/keepalived/samples/keepalived.conf.fwmark
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.sync
/usr/share/doc/keepalived/samples/keepalived.conf.SMTP_CHECK
/usr/share/doc/keepalived/samples/keepalived.conf.HTTP_GET.port
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.scripts
/usr/share/doc/keepalived/samples/keepalived.conf.SSL_GET
/usr/share/doc/keepalived/samples/keepalived.conf.virtual_server_group
/usr/share/doc/keepalived/samples/keepalived.conf.virtualhost
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.static_ipaddress
/usr/share/doc/keepalived/samples/keepalived.conf.misc_check
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.localcheck
/usr/share/doc/keepalived/samples/keepalived.conf.sample
/usr/share/doc/keepalived/samples/keepalived.conf.misc_check_arg
/usr/share/doc/keepalived/samples/keepalived.conf.IPv6
/usr/share/doc/keepalived/samples/keepalived.conf.quorum
/usr/share/doc/keepalived/samples/keepalived.conf.inhibit
/usr/share/doc/keepalived/samples/keepalived.conf.track_interface
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.lvs_syncd
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp
/usr/share/doc/keepalived/samples/keepalived.conf.status_code
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp.rules
/usr/share/doc/keepalived/keepalived.conf.SYNOPSIS.gz

#
root@hk-master2:~# vim /etc/keepalived/keepalived.conf #hk-master1與這個(gè)配置就routid和優(yōu)先級(jí)不一樣其他的都一樣

! Configuration File for keepalived

global_defs {
   #notification_email {
    # acassen
   #}
 #  notification_email_from Alexandre.Cassen@firewall.loc
  # smtp_server 192.168.200.1
  # smtp_connect_timeout 30
   router_id LVS_DEVEL_114
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    garp_master_delay 10
    #smtp_alert
    virtual_router_id 51
    priority 99
    nopreempt
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.200 label ens33:1
    }
}

haproxy動(dòng)態(tài)上線下線后端服務(wù)器

#以上基于hk-master1實(shí)現(xiàn)了一個(gè)vip-192.168.1.200蛛砰。 這里基于這個(gè)vip做負(fù)載均衡配置
#hk-master1:
global
        log /dev/log    local0
        log /dev/log    local1 notice
        chroot /var/lib/haproxy
        nbproc 2
        maxconn 65536
        stats timeout 30s
        cpu-map 1 0
        cpu-map 2 1
        stats socket /run/haproxy/admin.sock1 mode 660 level admin process 1
        stats socket /run/haproxy/admin.sock2 mode 660 level admin process 2
        nbthread 12
        user haproxy
        group haproxy
        daemon
        #ulimit -n 65536

defaults
        log     global
        mode    http
        option  httplog
        option  dontlognull
        timeout connect 5000
        timeout client  50000
        timeout server  50000
        errorfile 400 /etc/haproxy/errors/400.http
        errorfile 403 /etc/haproxy/errors/403.http
        errorfile 408 /etc/haproxy/errors/408.http
        errorfile 500 /etc/haproxy/errors/500.http
        errorfile 502 /etc/haproxy/errors/502.http
        errorfile 503 /etc/haproxy/errors/503.http
        errorfile 504 /etc/haproxy/errors/504.http
listen stats
  mode http
  bind 192.168.1.113:9999
  stats enable
  log global
  stats uri /haproxy-status
  stats auth haadmin:123123

listen nginx
  bind 192.168.1.200:80
  mode http
  server 192.168.1.111 192.168.1.111:80 check inter 2s fall 3 rise 5
  server 192.168.1.112 192.168.1.112:80 check inter 2s fall 3 rise 5   
  
#升級(jí)前下線后端服務(wù)器
root@hk-master1:~# cat updatecode.sh 
#!/bin/bash
cpus=`cat /proc/cpuinfo |grep processor |wc -l`

for ((i=1;i<=$cpus;i++));  
  do
     echo "disable server $1/$2" | socat stdio /run/haproxy/admin.sock$i;
  done  

#升級(jí)完畢上線后端服務(wù)器
root@hk-master1:~# cat percode.sh 
#!/bin/bash
cpus=`cat /proc/cpuinfo |grep processor |wc -l`

for ((i=1;i<=$cpus;i++));  
  do
     echo "enable server $1/$2" | socat stdio /run/haproxy/admin.sock$i;
  done  

LVS+keepalived 高可用集群

lvs 主要的工作是提供調(diào)度算法，把客戶端請(qǐng)求按照需求調(diào)度在 real 服務(wù)器黍衙，keepalived 主要的工作是提供 lvs 控制器的一個(gè)冗余泥畅，并且對(duì) real 服務(wù)器做健康檢查，發(fā)現(xiàn)不健康的 real 服務(wù)器琅翻，就把它從 lvs 集群中剔除位仁，real 服務(wù)器只負(fù)責(zé)提供服務(wù)。

keepalived底層有關(guān)于IPVS的功能模塊方椎，可以直接在其配置文件中實(shí)現(xiàn)LVS的配置聂抢，不需要通過(guò)ipvsadm命令再單獨(dú)配置

LVS 負(fù)載策略介紹

#IP 負(fù)載均衡技術(shù)（VS/NAT,VS/TUN,VS/DR）：
Virtual Server via Network Address Translation（VS/NAT）
通過(guò)網(wǎng)絡(luò)地址轉(zhuǎn)換，調(diào)度器重寫請(qǐng)求報(bào)文的目標(biāo)地址棠众，根據(jù)預(yù)設(shè)的調(diào)度算法琳疏，將請(qǐng)求分派給后端的真實(shí)服務(wù)器；真實(shí)服務(wù)器的響應(yīng)報(bào)文通過(guò)調(diào)度器時(shí)闸拿，報(bào)文的源地址被重寫轿亮，再返回給客戶，完成整個(gè)負(fù)載調(diào)度過(guò)程胸墙。

Virtual Server via IP Tunneling（VS/TUN）
采用 NAT 技術(shù)時(shí)，由于請(qǐng)求和響應(yīng)報(bào)文都必須經(jīng)過(guò)調(diào)度器地址重寫按咒，當(dāng)客戶請(qǐng)求越來(lái)越多時(shí)迟隅，調(diào)度器的處理能力將成為瓶頸但骨。為了解決這個(gè)問(wèn)題，調(diào)度器把請(qǐng)求報(bào) 文通過(guò) IP 隧道轉(zhuǎn)發(fā)至真實(shí)服務(wù)器智袭，而真實(shí)服務(wù)器將響應(yīng)直接返回給客戶奔缠，所以調(diào)度器只處理請(qǐng)求報(bào)文。由于一般網(wǎng)絡(luò)服務(wù)應(yīng)答比請(qǐng)求報(bào)文大許多吼野，采用 VS/TUN 技術(shù)后校哎，集群系統(tǒng)的最大吞吐量可以提高 10 倍。

Virtual Server via Direct Routing（VS/DR）
VS/DR 通過(guò)改寫請(qǐng)求報(bào)文的 MAC 地址瞳步，將請(qǐng)求發(fā)送到真實(shí)服務(wù)器闷哆，而真實(shí)服務(wù)器將響應(yīng)直接返回給客戶。同 VS/TUN 技術(shù)一樣单起，VS/DR 技術(shù)可極大地 提高集群系統(tǒng)的伸縮性抱怔。這種方法沒(méi)有 IP 隧道的開(kāi)銷，對(duì)集群中的真實(shí)服務(wù)器也沒(méi)有必須支持 IP 隧道協(xié)議的要求嘀倒，但是要求調(diào)度器與真實(shí)服務(wù)器都有一塊網(wǎng)卡連 在同一物理網(wǎng)段上屈留。

LVS 調(diào)度算法

1.輪詢：Round Robin，簡(jiǎn)稱rr测蘑，分發(fā)器按照循環(huán)的方式將請(qǐng)求平均的發(fā)送給后端的rs

2.加權(quán)輪詢：Weight Round-Robin灌危，簡(jiǎn)稱wrr，增對(duì)輪詢的優(yōu)化碳胳，會(huì)給每臺(tái)rs定義對(duì)應(yīng)的權(quán)重值勇蝙，權(quán)重值大的rs會(huì)比權(quán)重值小的rs接收到更多分發(fā)器轉(zhuǎn)發(fā)的請(qǐng)求

3.最小連接：Least-Connection，簡(jiǎn)稱lc固逗，分發(fā)器向每臺(tái)rs轉(zhuǎn)發(fā)請(qǐng)求時(shí)浅蚪，會(huì)記錄rs的連接數(shù)，根據(jù)連接數(shù)判斷所有rs的情況烫罩，將最新的請(qǐng)求轉(zhuǎn)發(fā)給連接數(shù)最少的rs

4.加權(quán)最小連接：Weight Least-Connection惜傲，簡(jiǎn)稱wlc，增對(duì)最小連接的優(yōu)化贝攒，定義每臺(tái)rs的權(quán)重值盗誊，分發(fā)器將新的請(qǐng)求轉(zhuǎn)發(fā)給rs時(shí)，會(huì)根據(jù)權(quán)重值判斷轉(zhuǎn)發(fā)請(qǐng)求給每臺(tái)rs的比例隘弊，分發(fā)器可以自動(dòng)判斷rs的情況哈踱，動(dòng)態(tài)調(diào)整權(quán)重值

#以上為4中常用調(diào)度算法，除此之外還有基于局部性的最小連接梨熙、帶復(fù)制的基于局部性最小連接开镣、目標(biāo)地址散列調(diào)度、源地址散列調(diào)度等

LVS NAT模式搭建

測(cè)試環(huán)境：準(zhǔn)備3臺(tái)機(jī)器咽扇，1臺(tái)分發(fā)器(dir)和2臺(tái)rs

dir內(nèi)網(wǎng)：192.168.1.113 外網(wǎng)：192.168.111.200
rs1內(nèi)網(wǎng)：192.168.1.111
rs2內(nèi)網(wǎng)：192.168.1.112

apt install -y iptables #all node
systemctl enable iptables --now #all node
#設(shè)置rs1與rs2的網(wǎng)關(guān)為dir的內(nèi)網(wǎng)ip：
root@hk-slave1:/opt# vim /etc/netplan/01-netcfg.yaml 
root@hk-slave1:/opt# netplan apply
root@hk-slave1:/opt# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    0      0        0 ens33
0.0.0.0         192.168.1.113   0.0.0.0         UG    0      0        0 ens33
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 ens33
# dir 上創(chuàng)建vip 開(kāi)啟代理后端服務(wù)的lvs 模塊
root@hk-master1:~# cat  /usr/local/sbin/lvs_nat.sh
#! /bin/bash
# dir上開(kāi)啟路由轉(zhuǎn)發(fā)功能
echo 1 > /proc/sys/net/ipv4/ip_forward
# 關(guān)閉icmp的重定向
echo 0 > /proc/sys/net/ipv4/conf/all/send_redirects
echo 0 > /proc/sys/net/ipv4/conf/default/send_redirects
# 注意區(qū)分網(wǎng)卡名字邪财，兩個(gè)網(wǎng)卡分別為ens33(內(nèi)網(wǎng))和ens37(外網(wǎng))
echo 0 > /proc/sys/net/ipv4/conf/ens33/send_redirects
#echo 0 > /proc/sys/net/ipv4/conf/ens37/send_redirects
# dir設(shè)置nat防火墻
iptables -t nat -F
iptables -t nat -X
iptables -t nat -A POSTROUTING -s 192.168.1.0/24  -j MASQUERADE
# 定義ipvsadm路徑到變量
IPVSADM='/sbin/ipvsadm'
# 清空ipvsadm規(guī)則
$IPVSADM -C
# 配置lvs陕壹，-s 指定調(diào)度算法為輪詢
$IPVSADM -A -t 192.168.1.200:80 -s rr
# -r指定rs1，-w表示模式為nat树埠，-w指定權(quán)重值
$IPVSADM -a -t 192.168.1.200:80 -r 192.168.1.111:80 -m -w 1
# 指定rs2
$IPVSADM -a -t 192.168.1.200:80 -r 192.168.1.112:80 -m -w 1
root@hk-master1:~# bash /usr/local/sbin/lvs_nat.sh 
# 驗(yàn)證代理配置
root@hk-master1:~# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.200:80 rr
  -> 192.168.1.111:80             Masq    1      0          0         
  -> 192.168.1.112:80             Masq    1      0          0   
  
 #dir 主機(jī) curl 調(diào)用虛擬ip 192.168.1.200
 root@hk-master1:~# curl 192.168.1.200
192.168.1.112 nginx page
root@hk-master1:~# curl 192.168.1.200
192.168.1.111 nginx page
root@hk-master1:~# curl 192.168.1.200
192.168.1.112 nginx page
root@hk-master1:~# curl 192.168.1.200
192.168.1.111 nginx page
root@hk-master1:~# 

LVS DR 模式搭建

測(cè)試環(huán)境：準(zhǔn)備3臺(tái)機(jī)器糠馆，1臺(tái)分發(fā)器(dir)和2臺(tái)rs

dir內(nèi)網(wǎng)：192.168.1.113
rs1內(nèi)網(wǎng)：192.168.1.111
rs2內(nèi)網(wǎng)：192.168.1.112
VIP：192.168.1.200

DR模式rs1，與rs2機(jī)器的網(wǎng)關(guān)不需要配置為dir的ip地址怎憋，同樣使用iptables工具管理防火墻又碌，也要下載ipvsadm

root@hk-master1:~#iptables -F 
root@hk-master1:~# ipvsadm -C
root@hk-master1:~# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

root@hk-master1:~# bash /usr/local/sbin/lvs_dr.sh 
SIOCADDRT: File exists

root@hk-master1:~# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.200:80 rr
  -> 192.168.1.111:80             Route   1      0          0         
  -> 192.168.1.112:80             Route   1      0          0         

root@hk-master1:~# cat /usr/local/sbin/lvs_dr.sh 
#! /bin/bash
# 開(kāi)啟端口轉(zhuǎn)發(fā)
echo 1 > /proc/sys/net/ipv4/ip_forward
ipv=/sbin/ipvsadm
vip=192.168.1.200
rs1=192.168.1.111
rs2=192.168.1.112
# 添加VIP
ifconfig ens33:2 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev ens33:2
# 清空ipvsadm規(guī)則
$ipv -C
# 定義lvs調(diào)度算法為輪詢
$ipv -A -t $vip:80 -s rr
# 指定轉(zhuǎn)發(fā)目標(biāo)rs1，-g表示dr模式绊袋，-w定義權(quán)重值
$ipv -a -t $vip:80 -r $rs1:80 -g -w 1
# 指定轉(zhuǎn)發(fā)目標(biāo)rs2
$ipv -a -t $vip:80 -r $rs2:80 -g -w 1

# rs1和rs2 都執(zhí)行：
root@hk-slave1:/opt# bash /usr/local/sbin/lvs_rs.sh 

root@hk-slave1:/opt# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.1.200/32 brd 192.168.1.200 scope global lo:0
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/enser 00:0c:29:9f:37:c7 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.111/24 brd 192.168.1.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe9f:37c7/64 scope link 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/enser 02:42:d9:b9:ab:13 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:d9ff:feb9:ab13/64 scope link 
       valid_lft forever preferred_lft forever

root@hk-slave1:/opt# cat /usr/local/sbin/lvs_rs.sh 
#!/bin/bash
netplan apply
vip=192.168.1.200
# 把vip綁定在lo上毕匀，是為了實(shí)現(xiàn)rs直接把結(jié)果返回給客戶端
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
# 以下操作為更改arp內(nèi)核參數(shù)，目的是為了讓rs順利發(fā)送mac地址給客戶端
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce

結(jié)合keepalive 實(shí)現(xiàn)高可用

以上面配置的LVS DR模式為例愤炸，使用keepalived+lvs的場(chǎng)景：

1.dir會(huì)將收到的請(qǐng)求分發(fā)給后端的rs期揪，但是當(dāng)某臺(tái)rs宕機(jī)的時(shí)候，dir不會(huì)知道规个，還會(huì)繼續(xù)分發(fā)請(qǐng)求到宕機(jī)的rs機(jī)器凤薛，為了避免該情況出現(xiàn)，可以使用keepalived的避免

#清空規(guī)則
root@hk-master1:~# iptables -F
root@hk-master1:~# ipvsadm -C
#編輯keepalive配置文件
root@hk-master1:~# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.200:80 rr persistent 1
  -> 192.168.1.111:80             Route   100    0          0         
  -> 192.168.1.112:80             Route   100    0          0         
root@hk-master1:~# vim /etc/keepalived/keepalived.conf 
root@hk-master1:~# cat /etc/keepalived/keepalived.conf 
vrrp_instance VI_1 {
    state BACKUP
    #綁定vip的網(wǎng)卡
    interface ens33
    #路由id诞仓，需要與backup機(jī)器相同
    virtual_router_id 51
    #定義權(quán)重缤苫，備用服務(wù)器上要小于100
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass keepalived123
    }
    virtual_ipaddress {
        192.168.1.200
    }
}
virtual_server 192.168.1.200 80 {
    #每隔10秒查詢r(jià)ealserver狀態(tài)
    delay_loop 10
    #lvs 算法
    lb_algo rr
    #DR模式
    lb_kind DR
    #同一IP的連續(xù)1秒內(nèi)被分配到同一臺(tái)rs
    persistence_timeout 1
    #用TCP協(xié)議檢查rs
    protocol TCP

    real_server 192.168.1.111 80 {
        #權(quán)重
        weight 100
        TCP_CHECK {
        #10秒無(wú)響應(yīng)超時(shí)
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.1.112 80 {
        weight 100
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
     }
}           

#驗(yàn)證配置
root@hk-master1:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/enser 00:0c:29:bb:35:0d brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.113/24 brd 192.168.1.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.1.200/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:febb:350d/64 scope link 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/enser 02:42:c9:56:f7:39 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
root@hk-master1:~# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.200:80 wlc persistent 1
  -> 192.168.1.111:80             Route   100    1          0   
  
  #瀏覽器測(cè)試
  #在rs當(dāng)中一臺(tái)服務(wù)停的時(shí)候會(huì)暫時(shí)出現(xiàn)訪問(wèn)不了的清空
curl: (7) Failed to connect to 192.168.1.200 port 80: Connection refused
root@hk-master2:~# curl 192.168.1.200
curl: (7) Failed to connect to 192.168.1.200 port 80: Connection refused
root@hk-master2:~# curl 192.168.1.200
192.168.1.112 nginx page
root@hk-master2:~# curl 192.168.1.200

2.完整的架構(gòu)dir需要兩臺(tái)，實(shí)現(xiàn)高可用墅拭，當(dāng)dir1宕機(jī)時(shí)活玲，dir2會(huì)切換為dir1，接收請(qǐng)求并分發(fā)到后端的rs

#配置一臺(tái)dir的從服務(wù)器
scp /etc/keepalived/keepalived.conf 192.168.1.114:/etc/keepalived/
root@hk-master2:~# cat /etc/keepalived/keepalived.conf 
vrrp_instance VI_1 {
    state BACKUP
    #綁定vip的網(wǎng)卡
    interface ens33
    #路由id谍婉，需要與backup機(jī)器相同
    virtual_router_id 51
    #定義權(quán)重舒憾，備用服務(wù)器上要小于100
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass keepalived123
    }
    virtual_ipaddress {
        192.168.1.200
    }
}
virtual_server 192.168.1.200 80 {
    #每隔10秒查詢r(jià)ealserver狀態(tài)
    delay_loop 10
    #lvs 算法
    lb_algo wlc
    #DR模式
    lb_kind DR
    #同一IP的連續(xù)1秒內(nèi)被分配到同一臺(tái)rs
    persistence_timeout 1
    #用TCP協(xié)議檢查rs
    protocol TCP

    real_server 192.168.1.111 80 {
        #權(quán)重
        weight 100
        TCP_CHECK {
        #10秒無(wú)響應(yīng)超時(shí)
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.1.112 80 {
        weight 100
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
     }
}

#重啟 rs 中的一臺(tái)nginx 測(cè)試keepalived
root@hk-master1:~# tail -f /var/log/syslog
Jun 26 20:55:17 k8s-node3 Keepalived_healthcheckers[46661]: TCP connection to [192.168.1.111]:tcp:80 failed.
Jun 26 20:55:17 k8s-node3 Keepalived_healthcheckers[46661]: Check on service [192.168.1.111]:tcp:80 failed after 1 retry.
Jun 26 20:55:17 k8s-node3 Keepalived_healthcheckers[46661]: Removing service [192.168.1.111]:tcp:80 to VS [192.168.1.200]:tcp:80
Jun 26 20:56:45 k8s-node3 Keepalived_healthcheckers[46661]: TCP connection to [192.168.1.111]:tcp:80 success.
Jun 26 20:56:45 k8s-node3 Keepalived_healthcheckers[46661]: Adding service [192.168.1.111]:tcp:80 to VS [192.168.1.200]:tcp:80
root@hk-master2:~# tail -f /var/log/syslog
Jun 26 20:55:15 k8s-node4 Keepalived_healthcheckers[43729]: Check on service [192.168.1.111]:tcp:80 failed after 1 retry.
Jun 26 20:55:15 k8s-node4 Keepalived_healthcheckers[43729]: Removing service [192.168.1.111]:tcp:80 to VS [192.168.1.200]:tcp:80
Jun 26 20:56:43 k8s-node4 Keepalived_healthcheckers[43729]: TCP connection to [192.168.1.111]:tcp:80 success.
Jun 26 20:56:43 k8s-node4 Keepalived_healthcheckers[43729]: Adding service [192.168.1.111]:tcp:80 to VS [192.168.1.200]:tcp:80

#測(cè)試 vip 飄逸
root@hk-master1:~# systemctl stop keepalived.service 
root@hk-master2:~# tail -f /var/log/syslog
Jun 26 20:57:36 k8s-node4 Keepalived_vrrp[43730]: VRRP_Instance(VI_1) Transition to MASTER STATE
Jun 26 20:57:37 k8s-node4 Keepalived_vrrp[43730]: VRRP_Instance(VI_1) Entering MASTER STATE
root@hk-master2:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/enser 00:0c:29:0f:45:99 brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.114/24 brd 192.168.1.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.1.200/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe0f:4599/64 scope link 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/enser 02:42:d1:34:f6:db brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever