1姨涡、說(shuō)明

2、準(zhǔn)備證書

2.1揍庄、下載ssl工具

  wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
  wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
  chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson 

2.2忿檩、準(zhǔn)備生成證書的模板文件

證書配置ca-config.json

{
  "signing": {
    "default": {
      "expiry": "8760h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "8760h"
      }
    }
  }
}

根證書ca-csr.json(etcd和kubernetes共用根證書，很多地方是分開(kāi)的)

{
    "CN": "etcd",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "Kubernetes",
        "OU": "Kubernetes-manual"
    }]
}

etcd-csr.json

{
    "CN": "kubernetes",
        "hosts": [
            $HOSTS$------可連接的地址
            "127.0.0.1"
        ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "Kubernetes",
        "OU": "Kubernetes-manual"
    }]
}

apiserver-csr.json

{
    "CN": "kube-apiserver",
        "hosts": [
          "10.20.16.227",
          "10.20.16.228",
          "10.20.16.229",
          "10.20.16.214",
          "127.0.0.1",
          "10.254.0.1", #一定要加上后期域名服務(wù)器的IP介蛉，后續(xù)coredns用到
          "localhost",
          "kubernetes",
          "kubernetes.default",
          "kubernetes.default.svc",
          "kubernetes.default.svc.cluster",
          "kubernetes.default.svc.cluster.local"#一定要加上后期域名服務(wù)器的名稱
        ],

    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "Kubernetes",
        "OU": "Kubernetes-manual"
    }]
}

front-proxy-client-csr.json

{
    "CN": "front-proxy-client",
    "key": {
        "algo": "rsa",
        "size": 2048
    }
}

admin-csr.json

{
    "CN": "admin",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "system:masters",
        "OU": "Kubernetes-manual"
    }]
}

manager-csr.json

{
    "CN": "system:kube-controller-manager",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "system:kube-controller-manager",
        "OU": "Kubernetes-manual"
    }]
}

scheduler-csr.json

{
    "CN": "system:kube-scheduler",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "system:kube-scheduler",
        "OU": "Kubernetes-manual"
    }]
}

kubelet-csr.json

{
    "CN": "system:node:$NODE",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "L": "NanJing",
        "ST": "NanJing",
        "O": "system:nodes",
        "OU": "Kubernetes-manual"
    }]
}

proxy-csr.json

{
    "CN": "system:kube-proxy",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [{
        "C": "CN",
        "ST": "NanJing",
        "L": "NanJing",
        "O": "system:kube-proxy",
        "OU": "Kubernetes-manual"
    }]
}

2.2萌庆、根據(jù)模板文件生成證書和各個(gè)組件的配置文件（共用一個(gè)根證書）

生成證書

# 產(chǎn)生根證書
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
# etcd 證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  etcd-csr.json | cfssljson -bare etcd
# admin 證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  admin-csr.json | cfssljson -bare admin
# apiserver 證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  apiserver-csr.json | cfssljson -bare apiserver
# controller-manager證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  manager-csr.json | cfssljson -bare controller-manager
# scheduler證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  scheduler-csr.json | cfssljson -bare scheduler
# front-proxy-client證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  front-proxy-client-csr.json | cfssljson -bare front-proxy-client
# kubelet證書
# cfssl gencert \
#    -ca=ca.pem \
#    -ca-key=ca-key.pem \
#    -config=ca-config.json \
#    -hostname=$NODE \
#    -profile=kubernetes \
# kubelet-$NODE-csr.json | cfssljson -bare kubelet-$NODE
# scheduler證書
cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  proxy-csr.json | cfssljson -bare proxy
#--------------------------sa-----------------------------------
$ openssl genrsa -out sa.key 2048
$ openssl rsa -in sa.key -pubout -out sa.pub
$ ls sa.*
sa.key  sa.pub

生成配置文件

# 設(shè)置Master的地址
export KUBE_APISERVER="https://10.20.16.229:6443"
#--------------------------admin-----------------------------------
# admin set cluster
$ kubectl config set-cluster kubernetes \
    --certificate-authority=ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=./admin.conf

# admin set credentials
$ kubectl config set-credentials kubernetes-admin \
    --client-certificate=admin.pem \
    --client-key=admin-key.pem \
    --embed-certs=true \
    --kubeconfig=./admin.conf

# admin set context
$ kubectl config set-context kubernetes-admin@kubernetes \
    --cluster=kubernetes \
    --user=kubernetes-admin \
    --kubeconfig=./admin.conf

# admin set default context
$ kubectl config use-context kubernetes-admin@kubernetes \
    --kubeconfig=./admin.conf

#--------------------------manager-----------------------------------
# controller-manager set cluster
$ kubectl config set-cluster kubernetes \
    --certificate-authority=ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=./controller-manager.conf

# controller-manager set credentials
$ kubectl config set-credentials system:kube-controller-manager \
    --client-certificate=controller-manager.pem \
    --client-key=controller-manager-key.pem \
    --embed-certs=true \
    --kubeconfig=./controller-manager.conf

# controller-manager set context
$ kubectl config set-context system:kube-controller-manager@kubernetes \
    --cluster=kubernetes \
    --user=system:kube-controller-manager \
    --kubeconfig=./controller-manager.conf

# controller-manager set default context
$ kubectl config use-context system:kube-controller-manager@kubernetes \
    --kubeconfig=./controller-manager.conf


#--------------------------scheduler-----------------------------------
# scheduler set cluster
$ kubectl config set-cluster kubernetes \
    --certificate-authority=ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=./scheduler.conf

# scheduler set credentials
$ kubectl config set-credentials system:kube-scheduler \
    --client-certificate=scheduler.pem \
    --client-key=scheduler-key.pem \
    --embed-certs=true \
    --kubeconfig=./scheduler.conf

# scheduler set context
$ kubectl config set-context system:kube-scheduler@kubernetes \
    --cluster=kubernetes \
    --user=system:kube-scheduler \
    --kubeconfig=./scheduler.conf

# scheduler use default context
$ kubectl config use-context system:kube-scheduler@kubernetes \
    --kubeconfig=./scheduler.conf

#--------------------------proxy-----------------------------------
# proxy set cluster
$ kubectl config set-cluster kubernetes \
    --certificate-authority=ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=./proxy.conf

# proxy set credentials
$ kubectl config set-credentials system:kube-proxy \
    --client-certificate=proxy.pem \
    --client-key=proxy-key.pem \
    --embed-certs=true \
    --kubeconfig=./proxy.conf

# proxy set context
$ kubectl config set-context system:kube-proxy@kubernetes \
    --cluster=kubernetes \
    --user=system:kube-proxy \
    --kubeconfig=./proxy.conf

# proxy use default context
$ kubectl config use-context system:kube-proxy@kubernetes \
    --kubeconfig=./proxy.conf

#--------------------------kubelet-----------------------------------
# TLS 認(rèn)證需要每個(gè)節(jié)點(diǎn)都單獨(dú)產(chǎn)生證書（本次不采用）。 說(shuō)明如下：
#$ kubectl config set-cluster kubernetes \
#        --certificate-authority=ca.pem \
#       --embed-certs=true \
#        --server=${KUBE_APISERVER} \
#        --kubeconfig=./kubelet.conf 

#$ kubectl config set-cluster kubernetes \
#        --certificate-authority=ca.pem \
#        --embed-certs=true \
#        --server=${KUBE_APISERVER} \
#        --kubeconfig=./kubelet.conf 

#$ kubectl config set-credentials system:node:${NODE} \
#        --client-certificate=kubelet-${NODE}.pem \
#        --client-key=kubelet-${NODE}-key.pem \
#        --embed-certs=true \
#        --kubeconfig=./kubelet.conf

#$ kubectl config set-context system:node:${NODE}@kubernetes \
#       --cluster=kubernetes \
#        --user=system:node:${NODE} \
#      --kubeconfig=./kubelet.conf 
# $ kubectl config use-context system:node:${NODE}@kubernetes \
#        --kubeconfig=../kubelet.conf && \

# 通過(guò)TLS bootstrapping實(shí)現(xiàn)kube-apiserver動(dòng)態(tài)簽發(fā)證書（本次采用）币旧，
# 原理主要首先產(chǎn)生一個(gè)token践险，由kubelet向kube-apiserver發(fā)起證書申請(qǐng)請(qǐng)求，token一致后吹菱，動(dòng)態(tài)下發(fā)捏境。

# 產(chǎn)生令牌
export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
echo "BOOTSTRAP_TOKEN:"${BOOTSTRAP_TOKEN}
cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

$ kubectl config set-cluster kubernetes \
    --certificate-authority=ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=./bootstrap.conf

# bootstrap set credentials
$ kubectl config set-credentials kubelet-bootstrap \
    --token=${BOOTSTRAP_TOKEN} \
    --kubeconfig=./bootstrap.conf

# bootstrap set context
$ kubectl config set-context kubelet-bootstrap@kubernetes \
    --cluster=kubernetes \
    --user=kubelet-bootstrap \
    --kubeconfig=./bootstrap.conf

# bootstrap use default context
$ kubectl config use-context kubelet-bootstrap@kubernetes \
    --kubeconfig=./bootstrap.conf

3、文件配置service文件并啟動(dòng)相關(guān)組件

3.1毁葱、文件配置service

etcd.service

[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
#User=etcd
# set GOMAXPROCS to number of processors
# ExecStartPre=/bin/bash -c "GOMAXPROCS=$(nproc)"
ExecStart=/data/cloud/etcd/etcd \
--name=master \
--auto-tls=true \
--heartbeat-interval=1000 \
--election-timeout=5000 \
--client-cert-auth=true \
--ca-file=/data/cloud/ssl/ca.pem \
--cert-file=/data/cloud/ssl/etcd.pem \
--key-file=/data/cloud/ssl/etcd-key.pem \
--trusted-ca-file=/data/cloud/ssl/ca.pem \
--peer-client-cert-auth=true \
--peer-cert-file=/data/cloud/ssl/etcd.pem \
--peer-key-file=/data/cloud/ssl/etcd-key.pem \
--peer-trusted-ca-file=/data/cloud/ssl/ca.pem \
--initial-advertise-peer-urls=https://10.20.16.227:2380 \
--listen-peer-urls=https://10.20.16.227:2380 \
--listen-client-urls=https://10.20.16.227:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://10.20.16.227:2379 \
--initial-cluster-token=kubernetes \
--initial-cluster=node1=https://10.20.16.227:2380,node3=https://10.20.16.228:2380,master=https://10.20.16.229:2380 \
--initial-cluster-state=new \
--data-dir=/data/cloud/work/etcd \
--wal-dir=/data/cloud/work/etcd/wal
Restart=on-failure
LimitNOFILE=65536

kube-apiserver.service

[Unit]
Description=Kubernetes API Server
Documentation=http://kubernetes.io/docs/
After=network.target

[Service]
ExecStart=/data/cloud/kubernetes/bin/kube-apiserver \
    --v=2 \
    --log-dir=/data/cloud/work/kubernetes \
    --logtostderr=false \
    --allow-privileged=true \
    --bind-address=0.0.0.0 \
    --secure-port=6443 \
    --insecure-port=0 \
    --advertise-address=10.20.16.229 \
    --service-cluster-ip-range=10.254.0.0/16 \
    --service-node-port-range=30000-32767 \
    --etcd-servers=https://10.20.16.227:2379,https://10.20.16.228:2379,https://10.20.16.229:2379 \
    --etcd-cafile=/data/cloud/ssl/ca.pem \
    --etcd-certfile=/data/cloud/ssl/etcd.pem \
    --etcd-keyfile=/data/cloud/ssl/etcd-key.pem \
    --client-ca-file=/data/cloud/ssl/ca.pem \
    --tls-cert-file=/data/cloud/ssl/apiserver.pem \
    --tls-private-key-file=/data/cloud/ssl/apiserver-key.pem \
    --kubelet-client-certificate=/data/cloud/ssl/apiserver.pem \
    --kubelet-client-key=/data/cloud/ssl/apiserver-key.pem \
    --service-account-key-file=/data/cloud/ssl/sa.pub \
    --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
    --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
    --authorization-mode=Node,RBAC \
    --enable-bootstrap-token-auth=true \
    --token-auth-file=/data/cloud/ssl/token.csv \
    --requestheader-client-ca-file=/data/cloud/ssl/ca.pem \
    --proxy-client-cert-file=/data/cloud/ssl/front-proxy-client.pem \
    --proxy-client-key-file=/data/cloud/ssl/front-proxy-client-key.pem \
    --requestheader-allowed-names=aggregator \
    --requestheader-group-headers=X-Remote-Group \
    --requestheader-extra-headers-prefix=X-Remote-Extra- \
    --requestheader-username-headers=X-Remote-User \
    --audit-log-maxage=30 \
    --audit-log-maxbackup=3 \
    --audit-log-maxsize=100 \
    --audit-log-path=/data/cloud/work/kubernetes/audit.log \
    --event-ttl=1h 
 
Restart=on-failure
StartLimitInterval=0
RestartSec=10
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

kube-manager.service

[Unit]
Description=Kubernetes Controller Manager
Documentation=http://kubernetes.io/docs/

[Service]
ExecStart=/data/cloud/kubernetes/bin/kube-controller-manager \
  --address=127.0.0.1 \
  --allocate-node-cidrs=true \
  --cluster-cidr=172.16.0.0/16 \
  --cluster-name=kubernetes \
  --cluster-signing-cert-file=/data/cloud/ssl/ca.pem \
  --cluster-signing-key-file=/data/cloud/ssl/ca-key.pem \
  --service-account-private-key-file=/data/cloud/ssl/sa.key \
  --root-ca-file=/data/cloud/ssl/ca.pem \
  --leader-elect=true \
  --use-service-account-credentials=true \
  --node-monitor-grace-period=40s \
  --node-monitor-period=5s \
  --pod-eviction-timeout=2m0s \
  --controllers=*,bootstrapsigner,tokencleaner \
  --log-dir=/data/cloud/work/kubernetes/controller-manager \
  --logtostderr=false \
  --kubeconfig=/data/cloud/ssl/controller-manager.conf \
  --v=2
  
Restart=on-failure
RestartSec=5
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

kube-scheduler.service

[Unit]
Description=Kubernetes Scheduler
Documentation=http://kubernetes.io/docs/

[Service]
ExecStart=/data/cloud/kubernetes/bin/kube-scheduler \
  --address=127.0.0.1 \
  --leader-elect=true \
  --logtostderr=false \
  --log-dir=/data/cloud/work/kubernetes/kube-scheduler \
  --kubeconfig=/data/cloud/ssl/scheduler.conf \
  --v=2

Restart=on-failure
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

kubelet.service

[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/

[Service]
ExecStart=/data/cloud/kubernetes/bin/kubelet \
  --fail-swap-on=false \
  --hostname-override=node3 \
  --pod-infra-container-image=k8s.gcr.io/pause:3.1 \
  --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin \
  --kubeconfig=/data/cloud/ssl/kubelet.conf \
  --bootstrap-kubeconfig=/data/cloud/ssl/bootstrap.conf \
  --pod-manifest-path=/data/cloud/kubernetes/manifests \
  --allow-privileged=true \
  --cluster-dns=10.254.0.2 \
  --cluster-domain=cluster.kube. \
  --authorization-mode=Webhook \
  --client-ca-file=/data/cloud/ssl/ca.pem \
  --rotate-certificates=true \
  --cert-dir=/data/cloud/ssl \
  --cgroup-driver=cgroupfs \
  --serialize-image-pulls=false  \
  --root-dir=/data/cloud/work/kubernetes/kubelet \
  --log-dir=/data/cloud/work/kubernetes/kubelet \
  --logtostderr=false \
  --v=2

Restart=on-failure
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

kube-proxy.service

[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/

[Service]
ExecStart=/data/cloud/kubernetes/bin/kube-proxy \
  --bind-address=10.239.7.253 \
  --hostname-override=node4 \
  --kubeconfig=/data/cloud/pki/proxy.conf \
  --proxy-mode=iptables \
  --v=2 \
  --logtostderr=false \
  --log-file=proxy.log \
  --log-dir=/data/cloud/work/kubernetes/logs

Restart=on-failure
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

3.3垫言、安裝CNI插件

mkdir -p /etc/cni/net.d /opt/cni/bin
#拷貝模板用于保證kubelet啟動(dòng)時(shí)不報(bào)錯(cuò)（主要提示網(wǎng)絡(luò)插件為初始化），后續(xù)kube-router會(huì)重新寫入
cp 10-kuberouter.conf  /etc/cni/net.d
#到https://github.com/containernetworking/plugins下載相關(guān)包
tar xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin

10-kuberouter.conf

{"bridge":"kube-bridge","ipam":{"type":"host-local"},"isDefaultGateway":true,"name":"kubernetes","type":"bridge"}

3.3倾剿、啟動(dòng)相關(guān)組件

systemctl daemon-reload
systemctl enable etcd kube-apiserver kube-manager kube-scheduler kubelet 
systemctl start etcd kube-apiserver kube-manager kube-scheduler 
#創(chuàng)建clusterrolebinding 
kubectl create clusterrolebinding controller-node-clusterrolebing --clusterrole=system:kube-controller-manager  --user=system:kube-controller-manager
kubectl create clusterrolebinding scheduler-node-clusterrolebing  --clusterrole=system:kube-scheduler --user=system:kube-scheduler
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
kubectl create clusterrolebinding  kube-system-cluster-adminkube-system-cluster-admin --user=system:serviceaccount:kube-system:default  --clusterrole=cluster-admin

#啟動(dòng)kubelet
systemctl start kubelet

3.4 審核Node節(jié)點(diǎn)

為了方便操作在profile中做了alias

如果需要自動(dòng)審核筷频，移步CSR 請(qǐng)求自動(dòng)批準(zhǔn)

export ETCDCTL_API=3
alias etcdctl='etcdctl --endpoints=https://10.20.16.227:2379,https://10.20.16.228:2379,https://10.20.16.229:2379 --cacert=/data/cloud/ssl/ca.pem --cert=/data/cloud/ssl/etcd.pem --key=/data/cloud/ssl/etcd-key.pem'
alias kubectl='kubectl --kubeconfig=/data/cloud/ssl/admin.conf'

kubectl get csr | tail -n +2 | while read name number
do
    kubectl  certificate approve  $name 
done 

4.kube-router

4.1 說(shuō)明

kube-router組件取代kube-proxy,用lvs做svc負(fù)載均衡，更快穩(wěn)定前痘。
cluster-ip凛捏，external-ip 全網(wǎng)路由
解決iptables 性能和負(fù)載聚合問(wèn)題
還有iptables 負(fù)載NAT 丟失源ip問(wèn)題

4.2 參數(shù)說(shuō)明及部署

① Kube-router 從API server 獲取pods, services, endpoints, network policies等信息. 所有的細(xì)節(jié)信息都要通過(guò)得到API server. API server可以通過(guò)kube-router --master=http://192.168.1.99:8080/ 或者 kube-router --kubeconfig=<path to kubeconfig file>傳遞。

②如果想把kube-router 作為agent運(yùn)行在節(jié)點(diǎn)上芹缔，必須事先在每個(gè)安裝ipset (如果作為daemonset運(yùn)行時(shí)坯癣，鏡像中已經(jīng)預(yù)置ipset)。

③如果想通過(guò)kube-router實(shí)現(xiàn)pod-to-pod連接最欠，那么需要在controller manager中配置CIDRs用于各pod分配IP.通過(guò)如下參數(shù)設(shè)置：

--allocate-node-cidrs=true
--cluster-cidr=10.1.0.0/16 

④kube-router作為daemonset時(shí)示罗，需要 kube-apiserver設(shè)置

--allow-privileged=true

⑤如果想通過(guò)kube-router實(shí)現(xiàn)pod-to-pod連接必須安裝CNI 網(wǎng)絡(luò)插件，主要會(huì)用到bridge CNI plugin 和 host-local 兩個(gè)插件芝硬。配置https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/cni/10-kuberouter.conf

參數(shù)說(shuō)明

 --run-router=true 
 #啟用Pod網(wǎng)絡(luò) - 通過(guò)iBGP發(fā)布并學(xué)習(xí)到Pod的路由蚜点。 （默認(rèn)為true）
 --run-firewall=true
#啟用網(wǎng)絡(luò)策略 - 設(shè)置iptables為pod提供入口防火墻。 （默認(rèn)為true）
 --run-service-proxy=true 
#啟用服務(wù)代理 - 為Kubernetes服務(wù)設(shè)置IPVS拌阴。 （默認(rèn)為true）
--advertise-cluster-ip=true
#將該服務(wù)的集群IP添加到RIB绍绘，以便通告給BGP peers.
--advertise-external-ip=true
#將服務(wù)的外部IP添加到RIB，以便將其通告給BGP peers.
--cluster-asn=64512
#集群自身節(jié)點(diǎn)運(yùn)行iBGP的ASN編號(hào).
--peer-router-ips=10.129.6.8
#所有節(jié)點(diǎn)將對(duì)等的外部路由器的IP地址，并通告集群ip和pod cidr陪拘。 （默認(rèn)[]）
--peer-router-asns=64513
#集群節(jié)點(diǎn)將向其通告集群ip和節(jié)點(diǎn)的pid cidr的BGP peers的ASN編號(hào)厂镇。 （默認(rèn)[]）

部署 kubectl apply -f kube-router.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-router-cfg
  namespace: kube-system
  labels:
    tier: node
    k8s-app: kube-router
data:
  cni-conf.json: |
    {
      "name":"kubernetes",
      "type":"bridge",
      "bridge":"kube-bridge",
      "isDefaultGateway":true,
      "ipam": {
        "type":"host-local"
      }
    }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    k8s-app: kube-router
    tier: node
  name: kube-router
  namespace: kube-system
spec:
  template:
    metadata:
      labels:
        k8s-app: kube-router
        tier: node
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      serviceAccountName: kube-router
      serviceAccount: kube-router
      containers:
      - name: kube-router
        image: registry.ipscloud.com/kube-router:v1.0 #修改鏡像
#        imagePullPolicy: Always #注釋默認(rèn)使用本地已有的
        args:
        - --run-router=true
        - --run-firewall=true
        - --run-service-proxy=true
        - --kubeconfig=/var/lib/kube-router/kubeconfig
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        resources:
          requests:
            cpu: 250m
            memory: 250Mi
        securityContext:
          privileged: true
        volumeMounts:
        - name: lib-modules
          mountPath: /lib/modules
          readOnly: true
        - name: cni-conf-dir
          mountPath: /etc/cni/net.d
        - name: kubeconfig
          mountPath: /var/lib/kube-router/kubeconfig
        - name: run
          mountPath: /var/run/docker.sock
          readOnly: true
      initContainers:
      - name: install-cni
        image: registry.ipscloud.com/busybox:v1.0#修改鏡像
 #       imagePullPolicy: Always #注釋默認(rèn)使用本地已有的
        command:
        - /bin/sh
        - -c
        - set -e -x;
          if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
            cp /etc/kube-router/cni-conf.json ${TMP};
            mv ${TMP} /etc/cni/net.d/10-kuberouter.conf;
          fi
        volumeMounts:
        - name: cni-conf-dir
          mountPath: /etc/cni/net.d
        - name: kube-router-cfg
          mountPath: /etc/kube-router
      hostNetwork: true
      hostIPC: true
      hostPID: true
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
        operator: Exists
      volumes:
      - name: lib-modules
        hostPath:
          path: /lib/modules
      - name: cni-conf-dir
        hostPath:
          path: /etc/cni/net.d
      - name: run
        hostPath:
          path: /var/run/docker.sock
      - name: kube-router-cfg
        configMap:
          name: kube-router-cfg
      - name: kubeconfig
        hostPath:
          path: /data/cloud/ssl/admin.conf #修改成自己的認(rèn)證文件位置
       # configMap:
        #  name: kube-proxy
         # items:
         # - key: kubeconfig.conf
         #   path: kubeconfig
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-router
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: kube-router
  namespace: kube-system
rules:
  - apiGroups:
    - ""
    resources:
      - namespaces
      - pods
      - services
      - nodes
      - nodes/proxy
      - endpoints
    verbs:
      - list
      - get
      - watch
  - apiGroups:
    - "networking.k8s.io"
    resources:
      - networkpolicies
    verbs:
      - list
      - get
      - watch
  - apiGroups:
    - extensions
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: kube-router
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-router
subjects:
- kind: ServiceAccount
  name: kube-router
  namespace: kube-system

4.3 驗(yàn)證

Network Services Controller

#查看主機(jī)分配網(wǎng)段
[root@host229 ~]# kubectl get nodes -o json | jq '.items[] | .spec'
{
  "podCIDR": "172.16.1.0/24"
}
{
  "podCIDR": "172.16.3.0/24"
}
{
  "podCIDR": "172.16.2.0/24"
}
{
  "podCIDR": "172.16.0.0/24"
}

[root@host229 yaml]# kubectl run myip --image=cloudnativelabs/whats-my-ip --replicas=3 --port=8080
deployment.apps/myip created
[root@host229 yaml]# kubectl get pod -o wide
NAME                    READY     STATUS    RESTARTS   AGE       IP           NODE
myip-5fc5cf6476-jmjzh   1/1       Running   0          28s       172.16.0.2   host229
myip-5fc5cf6476-qh546   1/1       Running   0          28s       172.16.1.2   host227
myip-5fc5cf6476-z7ccm   1/1       Running   0          28s       172.16.2.2   host228
[root@host229 yaml]# kubectl expose deployment myip --port=8080 --target-port=8080 --type=NodePort 
#-------------------不同主機(jī)都可以訪問(wèn)------------------
[root@host229 yaml]# curl host227:31007
HOSTNAME:myip-5fc5cf6476-qh546 IP:172.16.1.2
[root@host229 yaml]# curl host228:31007
HOSTNAME:myip-5fc5cf6476-z7ccm IP:172.16.2.2
[root@host229 yaml]# curl host229:31007
HOSTNAME:myip-5fc5cf6476-qh546 IP:172.16.1.2
[root@host229 yaml]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.20.16.229:31007 rr
  -> 172.16.0.2:8080              Masq    1      0          0         
  -> 172.16.1.2:8080              Masq    1      0          1         
  -> 172.16.2.2:8080              Masq    1      0          0         
TCP  10.254.0.1:443 rr
  -> 10.20.16.229:6443            Masq    1      0          0         
TCP  10.254.62.77:8080 rr
  -> 172.16.0.2:8080              Masq    1      0          0         
  -> 172.16.1.2:8080              Masq    1      0          0         
  -> 172.16.2.2:8080              Masq    1      0          0 
 # ----------------------刪除svc，創(chuàng)建新的svc左刽，并設(shè)置session persistence--------
[root@host229 yaml]# kubectl get svc -o wide
NAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE       SELECTOR
kubernetes   ClusterIP   10.254.0.1     <none>        443/TCP          52m       <none>
myip         NodePort    10.254.62.77   <none>        8080:31007/TCP   4m        run=myip
[root@host229 yaml]# kubectl delete svc myip
service "myip" deleted
[root@host229 yaml]#  kubectl expose deployment myip --port=8080 --target-port=8080 --type=NodePort  --session-affinity=ClientIP
service/myip exposed
#------------------再次訪問(wèn)發(fā)現(xiàn)同一個(gè)終端每次請(qǐng)求到pod為同一個(gè)-------------
[root@host229 yaml]# curl 10.254.102.139:8080
HOSTNAME:myip-5fc5cf6476-z7ccm IP:172.16.2.2
[root@host229 yaml]# curl 10.254.102.139:8080
HOSTNAME:myip-5fc5cf6476-z7ccm IP:172.16.2.2
[root@host229 yaml]# curl 10.254.102.139:8080
HOSTNAME:myip-5fc5cf6476-z7ccm IP:172.16.2.2
[root@host229 yaml]# curl 10.254.102.139:8080
HOSTNAME:myip-5fc5cf6476-z7ccm IP:172.16.2.2

Network Policy Controller（基于iptables ）

 # 創(chuàng)建兩個(gè)命名空間production 和 staging
[root@host229 ~]# kubectl create namespace production
namespace/production created
[root@host229 ~]# kubectl create namespace staging
namespace/staging created
 # 在兩個(gè)命名空間中創(chuàng)建guestbook-all-in-one.yaml
 # yaml 樣例可到github的kubernetes的example中找捺信，見(jiàn)https://github.com/kubernetes/examples/tree/master/guestbook/all-in-one
[root@host229 yaml]# kubectl  -n production apply -f guestbook-all-in-one.yml 
service/redis-master created
deployment.apps/redis-master created
service/redis-slave created
deployment.apps/redis-slave created
service/frontend created
deployment.apps/frontend created
[root@host229 yaml]# kubectl  -n staging  apply -f guestbook-all-in-one.yml 
service/redis-master created
deployment.apps/redis-master created
service/redis-slave created
deployment.apps/redis-slave created
service/frontend created
deployment.apps/frontend created
[root@host229 ~]# kubectl get pod --all-namespaces -o wide
NAMESPACE     NAME                            READY     STATUS    RESTARTS   AGE       IP             NODE
kube-system   kube-router-49nc6               1/1       Running   0          2h        10.20.16.229   host229
kube-system   kube-router-lqzmq               1/1       Running   0          2h        10.20.16.228   host228
kube-system   kube-router-nd46r               1/1       Running   0          2h        10.20.16.227   host227
kube-system   kube-router-vvdp8               1/1       Running   0          2h        10.20.16.214   host214
production    frontend-56f7975f44-654lz       1/1       Running   0          43m       172.16.2.3     host228
production    frontend-56f7975f44-jlxcd       1/1       Running   0          43m       172.16.0.4     host229
production    frontend-56f7975f44-s5wrg       1/1       Running   0          43m       172.16.1.3     host227
production    redis-master-6b464554c8-cwbk2   1/1       Running   0          43m       172.16.3.2     host214
production    redis-slave-b58dc4644-lqgrm     1/1       Running   0          43m       172.16.3.3     host214
production    redis-slave-b58dc4644-wmqzc     1/1       Running   0          43m       172.16.0.3     host229
staging       frontend-56f7975f44-7kncx       1/1       Running   0          40m       172.16.0.5     host229
staging       frontend-56f7975f44-stvsq       1/1       Running   0          40m       172.16.1.5     host227
staging       frontend-56f7975f44-zm7km       1/1       Running   0          40m       172.16.2.5     host228
staging       redis-master-6b464554c8-rjl68   1/1       Running   0          40m       172.16.2.4     host228
staging       redis-slave-b58dc4644-744sz     1/1       Running   0          40m       172.16.3.4     host214
staging       redis-slave-b58dc4644-kv859     1/1       Running   0          40m       172.16.1.4     host227
# ----------- 相同命名空間-----------
[root@host229 ~]# kubectl -n production  exec -it frontend-56f7975f44-654lz ping 172.16.3.2 
PING 172.16.3.2 (172.16.3.2): 56 data bytes
64 bytes from 172.16.3.2: icmp_seq=0 ttl=62 time=0.535 ms
64 bytes from 172.16.3.2: icmp_seq=1 ttl=62 time=0.227 ms
64 bytes from 172.16.3.2: icmp_seq=2 ttl=62 time=0.231 ms
# ----------- 不相同命名空間-----------
[root@host229 ~]# kubectl -n production  exec -it frontend-56f7975f44-654lz ping 172.16.2.4
PING 172.16.2.4 (172.16.2.4): 56 data bytes
64 bytes from 172.16.2.4: icmp_seq=0 ttl=64 time=0.206 ms
64 bytes from 172.16.2.4: icmp_seq=1 ttl=64 time=0.085 ms
64 bytes from 172.16.2.4: icmp_seq=2 ttl=64 time=0.072 m

網(wǎng)絡(luò)的隔離策略

#允許所有入口和出口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-all
spec:
  podSelector: {}
  ingress:
  - {}
  egress:
  - {}
#禁止所有入口和出口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny
spec:
  podSelector: {}
  policyTypes:
   - Ingress
   - Egress
#開(kāi)放特定入口和出口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny
spec:
  podSelector: {}
  policyTypes:
  - Egress
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: test-network-policy
  namespace: default
spec:
  podSelector:
    matchLabels:
      role: db
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - ipBlock:
        cidr: 172.17.0.0/16
        except:
        - 172.17.1.0/24
    - namespaceSelector:
        matchLabels:
          project: myproject
    - podSelector:
        matchLabels:
          role: frontend
    ports:
    - protocol: TCP
      port: 6379
  egress:
  - to:
    - ipBlock:
        cidr: 10.0.0.0/24
    ports:
    - protocol: TCP
      port: 5978

Network Routes Controller

①啟用hairpin traffic
kubectl annotate service my-service "kube-router.io/service.hairpin="
②啟用DSR(DSR將僅適用于外部IP)
kubectl annotate service my-service "kube-router.io/service.dsr=tunnel"
③負(fù)載均衡調(diào)度算法（默認(rèn)輪詢）
#最少連接
kubectl annotate service my-service "kube-router.io/service.scheduler=lc"
#輪序
kubectl annotate service my-service "kube-router.io/service.scheduler=rr"
#hash
kubectl annotate service my-service "kube-router.io/service.scheduler=sh"
#目標(biāo)hash
kubectl annotate service my-service "kube-router.io/service.scheduler=dh"

5.Ingress

1.Ingress SLB后端只會(huì)掛載打標(biāo)了node-role.kubernetes.io/ingress=true的集群Node；
2.Ingress Pod只會(huì)被部署到打標(biāo)了node-role.kubernetes.io/ingress=true的集群Node悠反；
3.不建議將Ingress Pod部署在集群Master節(jié)點(diǎn)上残黑，因?yàn)镸aster節(jié)點(diǎn)承載著集群的所有管控服務(wù)馍佑，以避免集群接入流量過(guò)高時(shí)對(duì)管控服務(wù)造成影響斋否。

# 1.主機(jī)打標(biāo)簽
[root@host229 yaml]# kubectl label no  host214 node-role.kubernetes.io/ingress=true
node/host214 labeled
[root@host229 yaml]# kubectl label no  host227 node-role.kubernetes.io/ingress=true
node/host227 labeled
[root@host229 yaml]# kubectl label no  host228 node-role.kubernetes.io/ingress=true
node/host228 labeled

5.CoreDNS

5.1 安裝

<service_name>.<namespace>.svc.<domain>

confingmap中[kubernetes cluster.local 10.254.0.0/16]和Service中的clusterIP保持一個(gè)網(wǎng)段

apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
  labels:
      kubernetes.io/cluster-service: "true"
      addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
    addonmanager.kubernetes.io/mode: Reconcile
  name: system:coredns
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - services
  - pods
  - namespaces
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
    addonmanager.kubernetes.io/mode: EnsureExists
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
  labels:
      addonmanager.kubernetes.io/mode: EnsureExists
data:
  Corefile: |
    .:53 {
        errors
        log stdout
        health
        kubernetes cluster.kube 10.254.0.0/16,
        prometheus
        proxy . /etc/resolv.conf
        cache 30
    }
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: coredns
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/name: "CoreDNS"
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: coredns
  template:
    metadata:
      labels:
        k8s-app: coredns
    spec:
      serviceAccountName: coredns
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      containers:
      - name: coredns
        image: registry.ipscloud.com/coredns:latest
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 500m
            memory: 500Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: coredns
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/name: "CoreDNS"
spec:
  selector:
    k8s-app: coredns
  clusterIP: 10.254.0.2
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  - name: metrics
    port: 9153
    protocol: TCP

4.2 查看DNS日志檢查

[root@host-10-1-235-89 ssl] kubectl --kubeconfig=admin.conf -n kube-system logs coredns-77b9b6d8f4-jb4bc
.:53
2018/08/09 08:49:36 [INFO] CoreDNS-1.2.0
2018/08/09 08:49:36 [INFO] linux/amd64, go1.10.3, 2e322f6
CoreDNS-1.2.0
linux/amd64, go1.10.3, 2e322f6

4.3 測(cè)試

創(chuàng)建一個(gè)nginx的service和pod：kubectl --kubeconfig=admin.conf create -f /nginx.yaml

apiVersion: v1
kind: Pod
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  containers:
  - name: nginx
    image: registry.ipscloud.com/nginx:1.13.8
    ports:
    - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  ports:
  - port: 80
    targetPort: 80
    protocol: TCP
  selector:
    app: nginx

查看nginx的pod

[root@host-10-1-235-89 ssl]# kubectl --kubeconfig=admin.conf get pod
NAME           READY     STATUS    RESTARTS   AGE
nginx          1/1       Running   0          5m
nginx-master   1/1       Running   0          5m

問(wèn)題

1.Unable to register node "node1" with API server: nodes "node1" is forbidden: node "master" cannot modify node "node1

解決辦法：刪除apiserver之前給kubelet下發(fā)的證書

2.Error from server (Forbidden): Forbidden (user=kube-apiserver, verb=get, resource=nodes, subresource=proxy) ( pods/log coredns-7c9d9954f8-9cf6h)

解決辦法：
wget https://kairen.github.io/files/manual-v1.10/master/apiserver-to-kubelet-rbac.yml.conf -O apiserver-to-kubelet-rbac.yml
kubectl --kubeconfig=admin.conf -n kube-system create -f  apiserver-to-kubelet-rbac.yml
或者執(zhí)行如下內(nèi)容：
# This binding gives the kube-apiserver user full access to the kubelet API
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kube-apiserver-kubelet-api-admin
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubelet-api-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: kube-apiserver
---
# This role allows full access to the kubelet API
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kubelet-api-admin
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
  - ""
  resources:
  - nodes/proxy
  - nodes/log
  - nodes/stats
  - nodes/metrics
  - nodes/spec
  verbs:
  - "*"

3.replicasets.apps is forbidden: User "system:kube-controller-manager" cannot list replicasets.apps at the cluster scope

$ kubectl --kubeconfig=admin.conf create clusterrolebinding  controller-node-clusterrolebing   --clusterrole=system:controller:node-controller --user=system:kube-scheduler

4.error: unable to upgrade connection: Forbidden (user=kube-apiserver, verb=create, resource=nodes, subresource=proxy)
創(chuàng)建apiserver到kubelet的權(quán)限有的版本默認(rèn)添加

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:kube-apiserver-to-kubelet
rules:
  - apiGroups:
      - ""
    resources:
      - nodes/proxy
      - nodes/stats
      - nodes/log
      - nodes/spec
      - nodes/metrics
    verbs:
      - "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:kube-apiserver
  namespace: ""
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kube-apiserver-to-kubelet
subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: User
    name: kube-apiserver