Driver Volume Mounts
The node plugin needs direct access to the host for making block devices and/or filesystem mounts available to the Kubernetes kubelet.
The mount point used by the CSI driver must be set to Bidirectional to allow Kubelet on the host to see mounts created by the CSI driver container. See the example below:
containers:
- name: my-csi-driver
...
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: mountpoint-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: node-driver-registrar
...
volumeMounts:
- name: registration-dir
mountPath: /registration
volumes:
# This volume is where the socket for kubelet->driver communication is done
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/<driver-name>
type: DirectoryOrCreate
# This volume is where the driver mounts volumes
- name: mountpoint-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
# This volume is where the node-driver-registrar registers the plugin
# with kubelet
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry
type: Directory
查看集群CSI插件
在集群中查看
root@server29:~# ls /var/lib/kubelet/
device-plugins pki plugins plugins_registry
root@server29:~# ls /var/lib/kubelet/p
pki/ plugins/ plugins_registry/
root@server29:~# ls /var/lib/kubelet/plugins
com.seaweedfs.csi
root@server29:~# ls /var/lib/kubelet/plugins_registry/
seaweedfs-csi-driver-reg.sock
root@server29:~# ls /var/lib/kubelet/pki/
kubelet-client-2019-06-19-15-21-34.pem kubelet-client-current.pem
root@server29:~# ls /var/lib/kubelet/device-plugins/
DEPRECATION kubelet_internal_checkpoint kubelet.sock nvidia.sock rdma-sriov-dp.sock
該節(jié)點安裝了com.seaweedfs.csi的csi插件
查看
kubelet的啟動命令中的根目錄--root-dir=/data1/k8s/kubelet
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77
containers etc-hosts plugins volumes
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77/containers
zhangbo6-inpainting
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77/volumes/
kubernetes.io~empty-dir kubernetes.io~nfs kubernetes.io~secret
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77/plugins/
kubernetes.io~empty-dir
去查看pod對應(yīng)的掛載信息發(fā)現(xiàn)沒有掛載seaweedfs
其中 020bb12f-1da6-11eb-85b5-8446fe72fa77是pod對應(yīng)的uid
kubectl get pods -o custom-columns=PodName:.metadata.name,PodUID:.metadata.uid |grep 020bb12f-1da6-11eb-85b5-8446fe72fa77
qiantao@qiant ~ % kc get pod zhangbo6-zhangbo6-inpainting -oyaml |grep containerID
- containerID: docker://10be561c850486cad50d1f7ed12e0e5a93e40d424dd929ab00ecdc048479bf92
在節(jié)點上也能看到對應(yīng)的進程
root@server29:~# ps -aux |grep 10be561c850486cad50d1f7ed12e0e5a93e40d424dd929ab00ecdc048479bf92
root 36133 0.0 0.0 11788 5108 ? Sl Nov03 0:05 containerd-shim -namespace moby -workdir /data/k8s/k8s_docker/data/containerd/daemon/io.containerd.runtime.v1.linux/moby/10be561c850486cad50d1f7ed12e0e5a93e40d424dd929ab00ecdc048479bf92 -address /var/run/docker/containerd/containerd.sock -containerd-binary /data/services/k8s_docker-GPU-1.18.9.3/bin/containerd -runtime-root /var/run/docker/runtime-nvidia
另外一個例子
當我在pod中啟用
apiVersion: v1
kind: Pod
metadata:
labels:
pm-pod-db-id: "9529"
name: qiantao-test-busybox
namespace: default
spec:
containers:
- args:
- sleep
- "100000000"
image: busybox
imagePullPolicy: IfNotPresent
name: test-create4-8
volumeMounts:
- mountPath: /dev/shm
name: dshm
- mountPath: /data/netfs
name: weedfs-volume
nodeSelector:
ip: 10.26.133.29
volumes:
- name: weedfs-volume
persistentVolumeClaim:
claimName: qiantao-pvc-csi-data
- emptyDir:
medium: Memory
name: dshm
PVC的配置如下
qiantao@qiant ~ % kc get pvc qiantao-pvc-csi-data -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-06-18T11:05:21Z"
finalizers:
- kubernetes.io/pvc-protection
name: qiantao-pvc-csi-data
namespace: default
resourceVersion: "614229919"
selfLink: /api/v1/namespaces/default/persistentvolumeclaims/qiantao-pvc-csi-data
uid: 9a394cfb-b153-11ea-bcb7-ac1f6b244dd4
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1000Gi
selector:
matchLabels:
pv: qiantao-pv-csi-data1
storageClassName: seaweedfs-storage
volumeMode: Filesystem
volumeName: qiantao-pv-csi-data
status:
accessModes:
- ReadWriteMany
capacity:
storage: 1000Gi
phase: Bound
PV如下所示
qiantao@qiant ~ % kc get pv qiantao-pv-csi-data -o yaml
apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
pv.kubernetes.io/provisioned-by: com.seaweedfs.csi
creationTimestamp: "2020-06-18T11:05:21Z"
finalizers:
- kubernetes.io/pv-protection
- external-attacher/seaweedfs-csi-driver
labels:
pv: qiantao-pv-csi-data1
name: qiantao-pv-csi-data
resourceVersion: "614229916"
selfLink: /api/v1/persistentvolumes/qiantao-pv-csi-data
uid: 9a3276ac-b153-11ea-bcb7-ac1f6b244dd4
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 1000Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: qiantao-pvc-csi-data
namespace: default
resourceVersion: "614229912"
uid: 9a394cfb-b153-11ea-bcb7-ac1f6b244dd4
csi:
driver: seaweedfs-csi-driver
fsType: ext4
volumeHandle: csi-qiantao-storage
persistentVolumeReclaimPolicy: Retain
storageClassName: seaweedfs-storage
volumeMode: Filesystem
status:
phase: Bound
最終在pod使用時（pod uid=53fe1c36-1ff9-11eb-8a23-b008758cb2a8）
root@server29:~# tree /data1/k8s/kubelet/pods/53fe1c36-1ff9-11eb-8a23-b008758cb2a8/volumes/ -L 4
/data1/k8s/kubelet/pods/53fe1c36-1ff9-11eb-8a23-b008758cb2a8/volumes/
├── kubernetes.io~csi
│ └── qiantao-pv-csi-data
│ ├── mount
│ │ ├── 111
│ │ ├── 3333
│ │ ├── agedatastes
│ │ ├── beat_game
│ │ ├── data
│ └── vol_data.json
├── kubernetes.io~empty-dir
│ └── dshm
└── kubernetes.io~secret
└── default-token-6lktj
├── ca.crt -> ..data/ca.crt
├── namespace -> ..data/namespace
└── token -> ..data/token
可以看出使用了kubernetes.io~csi類型的存儲掛載源请，當掛載成功后顯示了掛載的目錄
使用命令看到的目錄與文件系統(tǒng)上一致
kc exec -it qiantao-test-busybox -- ls /data/netfs
