1.對(duì)ceph添加mon和mgr節(jié)點(diǎn)
1.1 添加mon節(jié)點(diǎn)
ceph-mon是原生具備自選舉以實(shí)現(xiàn)高可用機(jī)制的ceph服務(wù)纽门，節(jié)點(diǎn)數(shù)量通常為奇數(shù)跨释。
#在準(zhǔn)備添加的mon節(jié)點(diǎn)執(zhí)行，減少后面添加進(jìn)集群的時(shí)間
root@ceph-mon3:~# apt install ceph-mon -y
root@ceph-mon2:~# apt install ceph-mon -y
#添加，在部署節(jié)點(diǎn)執(zhí)行
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon2
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon3
#檢查
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: 3586e7d1-9315-44e5-85bd-6bd3787ce574
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 82m)
mgr: ceph-mgr1(active, since 7d)
osd: 20 osds: 20 up (since 2h), 20 in (since 7d)
data:
pools: 2 pools, 33 pgs
objects: 0 objects, 0 B
usage: 5.7 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 33 active+clean
cephadmin@ceph-deploy:~/ceph-cluster$ ceph quorum_status --format json-pretty
{
"election_epoch": 12,
"quorum": [
0,
1,
2
],
"quorum_names": [
"ceph-mon1",
"ceph-mon2",
"ceph-mon3"
],
"quorum_leader_name": "ceph-mon1", #當(dāng)前mon主節(jié)點(diǎn)
"quorum_age": 4959,
"features": {
"quorum_con": "4540138314316775423",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging"
]
},
"monmap": {
"epoch": 3,
"fsid": "3586e7d1-9315-44e5-85bd-6bd3787ce574",
"modified": "2023-11-03T02:15:29.548725Z",
"created": "2023-10-26T03:38:28.654596Z",
"min_mon_release": 16,
"min_mon_release_name": "pacific",
"election_strategy": 1,
"disallowed_leaders: ": "",
"stretch_mode": false,
"tiebreaker_mon": "",
"removed_ranks: ": "",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging"
],
"optional": []
},
"mons": [
{
"rank": 0, #節(jié)點(diǎn)等級(jí)
"name": "ceph-mon1", #節(jié)點(diǎn)名稱
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.20.20.221:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.20.20.221:6789",
"nonce": 0
}
]
},
"addr": "172.20.20.221:6789/0",
"public_addr": "172.20.20.221:6789/0", #監(jiān)聽(tīng)地址
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 1,
"name": "ceph-mon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.20.20.222:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.20.20.222:6789",
"nonce": 0
}
]
},
"addr": "172.20.20.222:6789/0",
"public_addr": "172.20.20.222:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 2,
"name": "ceph-mon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.20.20.223:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.20.20.223:6789",
"nonce": 0
}
]
},
"addr": "172.20.20.223:6789/0",
"public_addr": "172.20.20.223:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
}
]
}
}
#節(jié)點(diǎn)詳細(xì)信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mon dump
epoch 3
fsid 3586e7d1-9315-44e5-85bd-6bd3787ce574
last_changed 2023-11-03T02:15:29.548725+0000
created 2023-10-26T03:38:28.654596+0000
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:172.20.20.221:3300/0,v1:172.20.20.221:6789/0] mon.ceph-mon1
1: [v2:172.20.20.222:3300/0,v1:172.20.20.222:6789/0] mon.ceph-mon2
2: [v2:172.20.20.223:3300/0,v1:172.20.20.223:6789/0] mon.ceph-mon3
dumped monmap epoch 3
擴(kuò)展
#查看節(jié)點(diǎn)ceph.conf文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = 3586e7d1-9315-44e5-85bd-6bd3787ce574
public_network = 172.20.20.0/24
cluster_network = 192.168.20.0/24
mon_initial_members = ceph-mon1
mon_host = 172.20.20.221 #只有一個(gè)節(jié)點(diǎn)地址
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
#修改文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = 3586e7d1-9315-44e5-85bd-6bd3787ce574
public_network = 172.20.20.0/24
cluster_network = 192.168.20.0/24
mon_initial_members = ceph-mon1,ceph-mon2,ceph-mon3
mon_host = 172.20.20.221,172.20.20.222,172.20.20.223 #添加其它的節(jié)點(diǎn)地址才能做到高可用，否則之前那個(gè)mon節(jié)點(diǎn)掛掉后，整個(gè)ceph集群將無(wú)法使用弟劲。
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
#把文件分發(fā)給各個(gè)節(jié)點(diǎn)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy --overwrite-conf config push ceph-deploy ceph-mon{1,2,3} ceph-mgr{1,2} ceph-node{1,2,3,4}
1.2 添加mgr節(jié)點(diǎn)
#在準(zhǔn)備添加的mgr節(jié)點(diǎn)執(zhí)行，減少后面添加進(jìn)集群的時(shí)間
root@ceph-mgr2:~# apt install ceph-mgr -y
#添加姥芥，在部署節(jié)點(diǎn)執(zhí)行
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr2
#分發(fā)秘鑰和集群配置文件給ceph-mgr2節(jié)點(diǎn)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-mgr2
#檢查
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: 3586e7d1-9315-44e5-85bd-6bd3787ce574
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 88m)
mgr: ceph-mgr1(active, since 8d), standbys: ceph-mgr2
osd: 20 osds: 20 up (since 2h), 20 in (since 7d)
data:
pools: 2 pools, 33 pgs
objects: 0 objects, 0 B
usage: 5.7 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 33 active+clean
2.熟練賬戶的授權(quán)
Ceph使用cephx協(xié)議對(duì)客戶端進(jìn)行身份認(rèn)證兔乞。
cephx用于對(duì)ceph保存的數(shù)據(jù)進(jìn)行認(rèn)證訪問(wèn)授權(quán)，用于對(duì)訪問(wèn)ceph的請(qǐng)求進(jìn)行認(rèn)證和授權(quán)檢測(cè)凉唐，與mon通信的請(qǐng)求都要經(jīng)過(guò)ceph認(rèn)證通過(guò)庸追，也可以在mon節(jié)點(diǎn)關(guān)閉cephx認(rèn)證，但關(guān)閉認(rèn)證之后任何訪問(wèn)都將被允許台囱，因此無(wú)法保證數(shù)據(jù)的安全性淡溯。
2.1 授權(quán)流程
每個(gè) mon 節(jié)點(diǎn)都可以對(duì)客戶端進(jìn)行身份認(rèn)證并分發(fā)秘鑰，因此多個(gè) mon 節(jié)點(diǎn)就不存在單點(diǎn)故障和認(rèn)證性能瓶頸簿训。
mon 節(jié)點(diǎn)會(huì)返回用于身份認(rèn)證的數(shù)據(jù)結(jié)構(gòu)咱娶，其中包含獲取 ceph 服務(wù)時(shí)用到的 session key,session key 通過(guò)客戶端秘鑰進(jìn)行加密傳輸，而秘鑰是在客戶端提前配置好的强品，保存在/etc/ceph/ceph.client.admin.keyring 文件中膘侮。
客戶端使用 session key 向 mon 請(qǐng)求所需要的服務(wù)，mon 向客戶端提供一個(gè) ticket的榛，用于向?qū)嶋H處理數(shù)據(jù)的 OSD 等服務(wù)驗(yàn)證客戶端身份琼了，MON 和 OSD 共享同一個(gè) secret，因此OSD 會(huì)信任所有 MON 發(fā)放的 ticket夫晌。
ticket 存在有效期雕薪，過(guò)期后重新發(fā)放。
注意：
??CephX 身份驗(yàn)證功能僅限制在 Ceph 的各組件之間慷丽，不能擴(kuò)展到其他非 ceph 組件
??Ceph 只負(fù)責(zé)認(rèn)證授權(quán)蹦哼，不能解決數(shù)據(jù)傳輸?shù)募用軉?wèn)題
image.png
2.2 訪問(wèn)流程
無(wú)論ceph客戶端是哪種類型鳄哭，例如塊設(shè)備要糊、對(duì)象存儲(chǔ)、文件系統(tǒng)妆丘，ceph都會(huì)在存儲(chǔ)池中將所有數(shù)據(jù)存儲(chǔ)為對(duì)象：
- ceph用戶需要擁有存儲(chǔ)池訪問(wèn)權(quán)限锄俄，才能讀取和寫入數(shù)據(jù)；
- ceph用戶必須擁有執(zhí)行權(quán)限才能使用ceph 的管理命令勺拣。
image.png
image.png
2.3 ceph用戶
用戶是指?jìng)€(gè)人(ceph 管理者)或系統(tǒng)參與者(MON/OSD/MDS)奶赠。通過(guò)創(chuàng)建用戶，可以控制用戶或哪個(gè)參與者能夠訪問(wèn)ceph存儲(chǔ)集群药有、以及可訪問(wèn)的存儲(chǔ)池及存儲(chǔ)池中的數(shù)據(jù)毅戈。
ceph支持多種類型的用戶苹丸，但可管理的用戶都屬于client類型。區(qū)分用戶類型的原因在于苇经，MON/OSD/MDS等系統(tǒng)組件特使用cephx協(xié)議赘理，但是它們?yōu)榉强蛻舳恕?/p>
通過(guò)點(diǎn)號(hào)分割用戶類型和用戶名，格式為TYPE.ID扇单，例如：client.admin商模。
cephadmin@ceph-deploy:~/ceph-cluster$ cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
key = AQA13zllZcdrExAACmK0yJUR6nHeowCTJPrlFQ==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
#列出指定的用戶信息，命令：ceph auth get 類型.用戶名
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get osd.10
[osd.10]
key = AQCmAjplu3PNOhAAsNOIby+qsdq2y4oWs79Rnw==
caps mgr = "allow profile osd"
caps mon = "allow profile osd"
caps osd = "allow *"
exported keyring for osd.10
2.4 ceph授權(quán)和使能
ceph基于使能/能力（caps）來(lái)描述用戶可針對(duì)MON/OSD或MDS使用的授權(quán)范圍或級(jí)別蜘澜。能力也用于限制對(duì)某一存儲(chǔ)池內(nèi)的數(shù)據(jù)或某個(gè)命名空間的訪問(wèn)施流。 Ceph 管理員用戶可在創(chuàng)建或更新普通用戶時(shí)賦予他相應(yīng)的能力。
Ceph 把數(shù)據(jù)以對(duì)象的形式存于各存儲(chǔ)池中鄙信。Ceph 用戶必須具有訪問(wèn)存儲(chǔ)池的權(quán)限才能夠讀寫數(shù)據(jù)瞪醋。另外，Ceph 用戶必須具有執(zhí)行權(quán)限才能夠使用 Ceph 的管理命令装诡。
通常的語(yǔ)法格式為：
daemon-type ‘a(chǎn)llow caps’ [...]
能力一覽表：
- r: 向用戶授予讀取權(quán)限趟章。訪問(wèn)監(jiān)視器(mon)以檢索CRUSH 運(yùn)行圖時(shí)需具有此能力。
- w: 向用戶授予針對(duì)對(duì)象的寫入權(quán)限慎王。
- x: 授予用戶調(diào)用類方法（包括讀取和寫入）的能力蚓土，以及在監(jiān)視器中執(zhí)行auth操作的能力。
- *: 授予用戶對(duì)特定守護(hù)進(jìn)程/存儲(chǔ)池的讀取赖淤、寫入和執(zhí)行權(quán)限蜀漆，以及執(zhí)行管理命令的能力。
- class-read: 授予用戶調(diào)用類讀取方法的能力咱旱，屬于是x能力的子集.
- class-write:授予用戶調(diào)用類寫入方法的能力确丢，屬于是x能力的子集.
- profile osd:授予用戶以某個(gè)OSD身份連接到其他OSD或監(jiān)視器的權(quán)限.授予OSD權(quán)限，使OSD能夠處理復(fù)制檢測(cè)信號(hào)流量和狀態(tài)報(bào)告(獲取OSD的狀態(tài)信息).
- profile mds:授予用戶以某個(gè)MDS身份連接到其他MDS或監(jiān)視器的權(quán)限.
- profile bootstrap-osd:授予用戶引導(dǎo)OSD的權(quán)限(初始化OSD并將OSD加入ceph 集群),授權(quán)給部署工具吐限，使其在引導(dǎo)OSD時(shí)有權(quán)添加密鑰鲜侥。
- profile bootstrap-mds:授予用戶引導(dǎo)元數(shù)據(jù)服務(wù)器的權(quán)限，授權(quán)部署工具權(quán)限诸典，使其在引導(dǎo)元數(shù)據(jù)服務(wù)器時(shí)有權(quán)添加密鑰.
MON能力
包括 r/w/x 和 allow profile cap(ceph 的運(yùn)行圖)
mon 'allow rwx'
mon 'allow profile osd'
OSD能力
包括r描函、w、x狐粱、class-read舀寓、class-write(類讀取）和 profile osd(類寫入)肌蜻，另外OSD能力還允許進(jìn)行存儲(chǔ)池和名稱空間設(shè)置互墓。
osd 'allow capability' [pool=poolname][namespace=namespace-name]
MDS能力
只需要 allow 或空都表示允許。
mds 'allow'
2.5 ceph 用戶管理
用戶管理功能可讓Ceph集群管理員能夠直接在Ceph集群中創(chuàng)建蒋搜、更新和刪除用戶篡撵。在Ceph集群中創(chuàng)建或刪除用戶時(shí)判莉，可能需要將密鑰分發(fā)到客戶端，以便將密鑰添加到密鑰環(huán)文件中/etc/ceph/ceph.client.admin.keyring育谬，此文件中可以包含一個(gè)或者多個(gè)用戶認(rèn)證信息骂租，凡是擁有此文件的節(jié)點(diǎn)，將具備訪問(wèn)ceph的權(quán)限斑司，而且可以使用其中任何一個(gè)賬戶的權(quán)限渗饮，此文件類似于linux系統(tǒng)的中的/etc/passwd文件。
2.5.1 列出用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth list
osd.0
key: AQAxAjpl3TiGFxAA4fFN2Q7InL6S1bBQL1+uBw==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.1
key: AQA9AjplxEH2HBAAX7WxXkZHIVr4XZbgOiQL0w==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.10
key: AQCmAjplu3PNOhAAsNOIby+qsdq2y4oWs79Rnw==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.11
key: AQCyAjplz3j7KRAA62nvKYskAcjpBYcAC1Xf2w==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.12
key: AQC+AjplHOfdFBAAHvWJkTck+S+ekDZDSLZ3Pw==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.13
key: AQDJAjpliv2RNhAAU+Hl9pwfXbqG6gwAdhBxog==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.14
key: AQDXAjplFV6dIBAAjwCdtxF3/CNH3ixiU8AJwA==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.15
key: AQDnAjpl3wYlCRAArSweuTQ+hBjCWlrLMzcvkQ==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.16
key: AQDyAjpl0PcXMBAAA7UNbnda5guuYhrzmSM69A==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.17
key: AQD+AjplEjegLhAAbFxguUvW1olMIbfg8HxVAA==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.18
key: AQAKAzplvqAXLxAAeyRqKIz6bWLN428aiNFD4A==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.19
key: AQAaAzplH36yJxAAbq0RBpTRHamvIm633/hzAA==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.2
key: AQBIAjplIP7UNhAALgO0LAjKBvbJR768Ka6JiQ==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.3
key: AQBUAjplRJ4uGRAAXCtOtq4xiEJgMMXurCfQ5Q==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.4
key: AQBgAjplM8SLGhAAR4TcAGCBgQuM6pA7tiMmIw==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.5
key: AQBsAjpl/R+JAxAAY/xSuELdsu56FI0Cxm+31w==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.6
key: AQB4AjplLhL8FRAAqFV4myI8iCgEY4EyFQBbwQ==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.7
key: AQCDAjplNCD5NBAArc9gKSJ5og43UqBtUa7xcw==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.8
key: AQCPAjplaqzZHBAAZcTO5G06osJBbDe6uAWWDA==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
osd.9
key: AQCbAjplbVROCBAAc7G/08tg74xRQ+STzWSngw==
caps: [mgr] allow profile osd
caps: [mon] allow profile osd
caps: [osd] allow *
client.admin
key: AQA13zllZcdrExAACmK0yJUR6nHeowCTJPrlFQ==
caps: [mds] allow *
caps: [mgr] allow *
caps: [mon] allow *
caps: [osd] allow *
client.bootstrap-mds
key: AQA13zllc+FrExAA1pzQbY32/HFdw/AbJw3DLg==
caps: [mon] allow profile bootstrap-mds
client.bootstrap-mgr
key: AQA13zllw/hrExAAzs+duofYhKx0u7m3F4APeQ==
caps: [mon] allow profile bootstrap-mgr
client.bootstrap-osd
key: AQA13zllDg5sExAAWo8Malb6IeGnSgjwWvy09Q==
caps: [mon] allow profile bootstrap-osd
client.bootstrap-rbd
key: AQA13zllAiRsExAAvAhpQicNqZ14iFw8RDy2Bw==
caps: [mon] allow profile bootstrap-rbd
client.bootstrap-rbd-mirror
key: AQA13zllVzlsExAAhAx2+wuOyYJkK8TUMDTZqA==
caps: [mon] allow profile bootstrap-rbd-mirror
client.bootstrap-rgw
key: AQA13zllpk5sExAAK+gSOjyg85ET0sPqtMMXfA==
caps: [mon] allow profile bootstrap-rgw
mgr.ceph-mgr1
key: AQBo4Dlln0vtHhAAuoQINJkKBjptRA2iYpsAJQ==
caps: [mds] allow *
caps: [mon] allow profile mgr
caps: [osd] allow *
mgr.ceph-mgr2
key: AQCJbERlcDhCNRAAPZ2uAz0wm6g2bMl3/d3nrQ==
caps: [mds] allow *
caps: [mon] allow profile mgr
caps: [osd] allow *
installed auth entries:
注意：TYPE.ID表示法
針對(duì)用戶采用TYPE.ID表示法宿刮，例如：osd.0指定是osd類并且ID為0的用戶(節(jié)點(diǎn))互站，client.admin是client類型的用戶，其ID為admin僵缺。
另外胡桃，每個(gè)用戶條目都有一個(gè) key: 對(duì)，一個(gè)或多個(gè) caps: 條目磕潮〈湟龋可以結(jié)合使用-o文件名選項(xiàng)和ceph auth list將輸出保存到某個(gè)文件。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth list -o auth.key
installed auth entries:
cephadmin@ceph-deploy:~/ceph-cluster$ ll auth.key
-rw-rw-r-- 1 cephadmin cephadmin 3810 Nov 3 06:44 auth.key
2.5.2 用戶管理
添加一個(gè)用戶會(huì)創(chuàng)建用戶名 (TYPE.ID)自脯、機(jī)密密鑰之景，以及包含在命令中用于創(chuàng)建該用戶的所有能力,用戶可使用其密鑰向 Ceph 存儲(chǔ)集群進(jìn)行身份驗(yàn)證。用戶的能力授予該用戶在Ceph monitor (mon)膏潮、Ceph OSD (osd) 或 Ceph 元數(shù)據(jù)服務(wù)器 (mds) 上進(jìn)行讀取锻狗、寫入或執(zhí)行的能力,可以使用以下幾個(gè)命令來(lái)添加用戶：
- ceph auth add
此命令是添加用戶的規(guī)范方法。它會(huì)創(chuàng)建用戶焕参、生成密鑰轻纪，并添加所有指定的能力。
#添加用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth add client.zhao mon 'allow r' osd 'allow rwx pool=mypool
> '
added key for client.zhao
#查看
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.zhao
[client.zhao]
key = AQAKmURlk2t7NxAA6gEARsGwMsbTXk2Bo2jznQ==
caps mon = "allow r"
caps osd = "allow rwx pool=mypool
"
exported keyring for client.zhao
- ceph auth get-or-create
此命令是創(chuàng)建用戶較為常見(jiàn)的方式之一叠纷，它會(huì)返回包含用戶名（在方括號(hào)中）和密鑰的格式刻帚，如果該用戶已存在，此命令以密鑰文件格式返回用戶名和密鑰信息涩嚣，還可以使用-o指定文件名選項(xiàng)輸出保存到指定文件中崇众。
#創(chuàng)建用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get-or-create client.jia mon 'allow r' osd 'allow rwx pool=mypool'
[client.jia]
key = AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
#驗(yàn)證用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.jia
[client.jia]
key = AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
caps mon = "allow r"
caps osd = "allow rwx pool=mypool"
exported keyring for client.jia
#再次創(chuàng)建用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get-or-create client.jia mon 'allow r' osd 'allow rwx pool=mypool'
[client.jia]
key = AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
- ceph auth get-or-create-key
此命令是創(chuàng)建用戶并僅返回用戶密鑰，對(duì)于只需要密鑰的客戶端（如libvirt) 缓艳，此命令非常有用校摩。如果該用戶已存在，此命令只返回用戶的密鑰阶淘，可以使用-o文件名選項(xiàng)將輸出保存到某個(gè)文件。
創(chuàng)建客戶端用戶時(shí)互妓，可以創(chuàng)建不具有能力的用戶溪窒，不具有能力的用戶可以進(jìn)行身份驗(yàn)證坤塞，但不能執(zhí)行其他操作，此類客戶端無(wú)法從監(jiān)視器檢索集群地圖澈蚌，但希望稍后再添加能力摹芙，可以使用ceph auth caps命令創(chuàng)建一個(gè)不具有能力的用戶。
典型的用戶至少對(duì)Ceph monitor具有讀取功能宛瞄，并對(duì) Ceph OSD具有讀取和寫入功能浮禾。此外，用戶的OSD權(quán)限通常限制為只能訪問(wèn)特定的存儲(chǔ)池份汗。
#只返回密鑰信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get-or-create-key client.jia mon 'allow r' osd 'allow rwx pool=mypool'
AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
- ceph auth print-key
只獲取單個(gè)指定用戶的key信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth print-key client.jia
AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
- 修改用戶能力
使用ceph auth caps命令可以指定用戶以及更改該用戶的能力盈电，設(shè)置新能力會(huì)完全覆蓋當(dāng)前的能力，因此要加上之前的用戶已經(jīng)擁有的能和新的能力杯活，如果看當(dāng)前能力匆帚，可以運(yùn)行ceph auth get USERTYPE.USERID，如果要添加能力旁钧，使用以下格式時(shí)還需要指定現(xiàn)有能力:
ceph auth caps USERTYPE.USERID daemon 'allow [r|w|x||...] [pool=pool-name] [namespace=namespace-name]' [daemon 'allow [r|w|x|]...] [pool=pool-name] [namespace=namespace-name]']
#查看用戶當(dāng)前權(quán)限
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth print-key client.jia
AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.jia
[client.jia]
key = AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
caps mon = "allow r"
caps osd = "allow rwx pool=mypool"
exported keyring for client.jia
#修改用戶權(quán)限
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth caps client.jia mon 'allow rw' osd 'allow rwx pool=mypool'
updated caps for client.jia
#查看修改后權(quán)限
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.jia
[client.jia]
key = AQAwmkRlpdTiBxAA74o7L3Ui+ROrf4Zsu+j1FQ==
caps mon = "allow rw" #權(quán)限變?yōu)閞w
caps osd = "allow rwx pool=mypool"
exported keyring for client.jia
- 刪除用戶
要?jiǎng)h除用戶使用ceph auth del TYPE.ID吸重，其中TYPE是client、osd歪今、mon或mds之一嚎幸，ID是用戶名或守護(hù)進(jìn)程的ID。
#刪除用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth del client.jia
updated
#再次查看用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.jia
Error ENOENT: failed to find client.jia in keyring
2.6 秘鑰環(huán)管理
ceph的秘鑰環(huán)是一個(gè)保存了secrets寄猩、keys鞭铆、certificates并且能夠讓客戶端通過(guò)認(rèn)證訪問(wèn)ceph的keyring file（集合文件），一個(gè)keyring file可以保存一個(gè)或多個(gè)認(rèn)證焦影，每一個(gè)key都有一個(gè)實(shí)體名稱加權(quán)限车遂，類型為{client|mon|mds|osd}.name。
當(dāng)客戶端訪問(wèn)ceph集群時(shí)斯辰，Ceph 客戶端會(huì)使用本地的 keyring 文件舶担。默認(rèn)使用下列路徑和名稱的 keyring 文件：
- /etc/ceph/<$cluster name>.<user $type>.<user $id>.keyring #保存單個(gè)用戶的keyring
- /etc/ceph/cluster.keyring #保存多個(gè)用戶的keyring
- /etc/ceph/keyring #未定義集群名稱的多個(gè)用戶的keyring
- /etc/ceph/keyring.bin #編譯后的二進(jìn)制文件
2.6.1 通過(guò)秘鑰環(huán)文件備份與恢復(fù)用戶
使用 ceph auth add 等命令添加的用戶還需要額外使用 ceph-authtool 命令為其創(chuàng)建用戶秘鑰環(huán)文件。
創(chuàng)建 keyring 文件命令格式：
ceph-authtool --create-keyring FILE
- 導(dǎo)出用戶認(rèn)證信息至keyring文件
#創(chuàng)建用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get-or-create client.user1 mon 'allow r' osd 'allow * pool=mypool'
[client.user1]
key = AQCHoERlshxOABAAOtXGN5QBJZJhX0c1QK2pkA==
#驗(yàn)證用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.user1
[client.user1]
key = AQCHoERlshxOABAAOtXGN5QBJZJhX0c1QK2pkA==
caps mon = "allow r"
caps osd = "allow * pool=mypool"
exported keyring for client.user1
#創(chuàng)建一個(gè)空的keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool --create-keyring ceph.client.user1.keyring
creating ceph.client.user1.keyring
#查看文件為空
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.client.user1.keyring
cephadmin@ceph-deploy:~/ceph-cluster$ file ceph.client.user1.keyring
ceph.client.user1.keyring: empty
#導(dǎo)出指定用戶keyring到指定文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.user1 -o ceph.client.user1.keyring
exported keyring for client.user1
#查看指定用戶的keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.client.user1.keyring
[client.user1]
key = AQCHoERlshxOABAAOtXGN5QBJZJhX0c1QK2pkA==
caps mon = "allow r"
caps osd = "allow * pool=mypool"
在創(chuàng)建包含單個(gè)用戶的秘鑰環(huán)時(shí)彬呻，通常建議使用<ceph集群名稱>.<用戶類型>.<用戶名>.keyring來(lái)命名衣陶，并將其保存至/etc/ceph目錄中。例如為client.user1用戶創(chuàng)建秘鑰環(huán)闸氮，命名為ceph.client.user1.keyring剪况。
- 從 keyring 文件恢復(fù)用戶認(rèn)證信息
可以使用ceph auth import -i {filename}指定keyring文件并導(dǎo)入到ceph，起到用戶備份和恢復(fù)的作用蒲跨。
#查看用戶認(rèn)證文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.client.user1.keyring
[client.user1]
key = AQCHoERlshxOABAAOtXGN5QBJZJhX0c1QK2pkA==
caps mon = "allow r"
caps osd = "allow * pool=mypool"
#刪除用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth del client.user1
updated
#確認(rèn)用戶被刪除
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.user1
Error ENOENT: failed to find client.user1 in keyring
#導(dǎo)入用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth import -i ceph.client.user1.keyring
imported keyring
#查看用戶已恢復(fù)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.user1
[client.user1]
key = AQCHoERlshxOABAAOtXGN5QBJZJhX0c1QK2pkA==
caps mon = "allow r"
caps osd = "allow * pool=mypool"
exported keyring for client.user1
2.6.2 秘鑰環(huán)文件多用戶
一個(gè)keyring文件中可以包含多個(gè)不同用戶的認(rèn)證文件囱晴。
將多用戶導(dǎo)出至密鑰環(huán)
#創(chuàng)建空的keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool --create-keyring ceph.client.user.keyring
creating ceph.client.user.keyring
#把a(bǔ)dmin用戶的keyring文件內(nèi)容導(dǎo)入到user用戶的keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool ./ceph.client.user.keyring --import-keyring ./ceph.client.admin.keyring
importing contents of ./ceph.client.admin.keyring into ./ceph.client.user.keyring
#驗(yàn)證keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool -l ./ceph.client.user.keyring
[client.admin]
key = AQA13zllZcdrExAACmK0yJUR6nHeowCTJPrlFQ==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
#再把user1的keyring導(dǎo)入
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool ./ceph.client.user.keyring --import-keyring ./ceph.client.user1.keyring
importing contents of ./ceph.client.user1.keyring into ./ceph.client.user.keyring
#查看user的keyring文件，包含多個(gè)用戶的認(rèn)證信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool -l ./ceph.client.user.keyring
[client.admin]
key = AQA13zllZcdrExAACmK0yJUR6nHeowCTJPrlFQ==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
[client.user1]
key = AQCHoERlshxOABAAOtXGN5QBJZJhX0c1QK2pkA==
caps mon = "allow r"
caps osd = "allow * pool=mypool"
3.基于普通用于掛載塊存儲(chǔ)去扣、實(shí)現(xiàn)對(duì)塊存儲(chǔ)的動(dòng)態(tài)空間拉伸
3.1 客戶端使用普通賬戶掛載并使用RBD
RBD（RADOS Block Devices）即塊存儲(chǔ)設(shè)備，RBD可以為KVM堪唐、VMware等虛擬化技術(shù)和云服務(wù)（OpenStack、kubernetes）提供高性能和無(wú)限可擴(kuò)展的存儲(chǔ)后端翎蹈，客戶端基于librbd庫(kù)即可將RADOS存儲(chǔ)集群用作塊設(shè)備淮菠，不過(guò)，用于rbd的存儲(chǔ)池需要事先啟用rbd功能并進(jìn)行初始化荤堪。
3.1.1 創(chuàng)建RBD
創(chuàng)建一個(gè)名為myrbd1的存儲(chǔ)池合陵，并在啟用rbd功能后對(duì)其進(jìn)行初始化。
#創(chuàng)建存儲(chǔ)池澄阳，指定pg和pgp的數(shù)量拥知，pgp是對(duì)存在于pg的數(shù)據(jù)進(jìn)行組合存儲(chǔ)，pgp通常等于pg的值
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create myrbd1 32 32
pool 'myrbd1' created
#查看
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool ls
device_health_metrics
mypool
myrbd1
#開(kāi)啟存儲(chǔ)池rbd功能
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool application enable myrbd1 rbd
enabled application 'rbd' on pool 'myrbd1'
#初始化存儲(chǔ)池
cephadmin@ceph-deploy:~/ceph-cluster$ rbd pool init -p myrbd1
3.1.2 創(chuàng)建和驗(yàn)證img
rbd存儲(chǔ)池并不能直接用于塊設(shè)備寇荧，而是需要事先在其中按需創(chuàng)建映像(image) ,并把映像文件作為塊設(shè)備使用, rbd命令可用于創(chuàng)建举庶、查看及刪除塊設(shè)備所在的映像(image),以及克隆映像、創(chuàng)建快照揩抡、將映像回滾到快照和查看快照等管理操作户侥。
#創(chuàng)建鏡像
cephadmin@ceph-deploy:~/ceph-cluster$ rbd create myimg1 --size 3G --pool myrbd1
cephadmin@ceph-deploy:~/ceph-cluster$ rbd create myimg2 --size 5G --pool myrbd1
#驗(yàn)證鏡像
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls --pool myrbd1
myimg1
myimg2
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls --pool myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg1 3 GiB 2
myimg2 5 GiB 2
#查看鏡像詳細(xì)信息
cephadmin@ceph-deploy:~/ceph-cluster$ rbd --image myimg1 --pool myrbd1 info
rbd image 'myimg1':
size 3 GiB in 768 objects
order 22 (4 MiB objects)
snapshot_count: 0
id: 3871e8bdaa8e
block_name_prefix: rbd_data.3871e8bdaa8e
format: 2
features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
op_features:
flags:
create_timestamp: Fri Nov 3 08:35:44 2023
access_timestamp: Fri Nov 3 08:35:44 2023
modify_timestamp: Fri Nov 3 08:35:44 2023
#以json格式顯示信息
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls --pool myrbd1 -l --format json --pretty-format
[
{
"image": "myimg1",
"id": "3871e8bdaa8e",
"size": 3221225472,
"format": 2
},
{
"image": "myimg2",
"id": "13598b03a766",
"size": 5368709120,
"format": 2
}
]
3.1.3 創(chuàng)建普通用戶并授權(quán)
#創(chuàng)建普通用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth add client.tom mon 'allow r' osd 'allow rwx pool=myrbd1'
added key for client.tom
#驗(yàn)證用戶信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.tom
[client.tom]
key = AQD9skRlZazPHhAAjNlHMTPgC3vgrn5bTqAiMQ==
caps mon = "allow r"
caps osd = "allow rwx pool=myrbd1"
exported keyring for client.tom
#創(chuàng)建keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool --create-keyring ceph.client.tom.keyring
creating ceph.client.tom.keyring
#導(dǎo)出用戶
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.tom -o ceph.client.tom.keyring
exported keyring for client.tom
#驗(yàn)證用戶keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.client.tom.keyring
[client.tom]
key = AQD9skRlZazPHhAAjNlHMTPgC3vgrn5bTqAiMQ==
caps mon = "allow r"
caps osd = "allow rwx pool=myrbd1"
3.1.4 安裝ceph客戶端，并同步相關(guān)認(rèn)證文件
#ubuntu安裝峦嗤，提前配置好ceph倉(cāng)庫(kù)
root@ceshi:~# apt install ceph-common -y
#同步認(rèn)證文件
cephadmin@ceph-deploy:~/ceph-cluster$ scp ceph.conf ceph.client.tom.keyring root@172.20.20.128:/etc/ceph/
#驗(yàn)證客戶端權(quán)限
root@ceshi:~# ceph --user tom -s
cluster:
id: 3586e7d1-9315-44e5-85bd-6bd3787ce574
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 8h)
mgr: ceph-mgr1(active, since 8d), standbys: ceph-mgr2
osd: 20 osds: 20 up (since 9h), 20 in (since 8d)
data:
pools: 4 pools, 97 pgs
objects: 96 objects, 143 MiB
usage: 5.9 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 97 active+clean
3.1.5 映射rbd
#映射rbd
root@ceshi:/etc# rbd --user tom -p myrbd1 map myimg2
/dev/rbd0
rbd: --user is deprecated, use --id
#驗(yàn)證rbd
root@ceshi:~# fdisk -l /dev/rbd0
Disk /dev/rbd0: 5 GiB, 5368709120 bytes, 10485760 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 65536 bytes / 65536 bytes
root@ceshi:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 55.7M 1 loop /snap/core18/2796
loop1 7:1 0 63.5M 1 loop /snap/core20/2015
loop2 7:2 0 63.5M 1 loop /snap/core20/1974
loop3 7:3 0 55.7M 1 loop /snap/core18/2790
loop4 7:4 0 70.3M 1 loop /snap/lxd/21029
loop5 7:5 0 91.9M 1 loop /snap/lxd/24061
loop6 7:6 0 40.9M 1 loop /snap/snapd/20290
loop7 7:7 0 40.9M 1 loop /snap/snapd/20092
sda 8:0 0 20G 0 disk
├─sda1 8:1 0 1M 0 part
├─sda2 8:2 0 1G 0 part /boot
└─sda3 8:3 0 19G 0 part
└─ubuntu--vg-ubuntu--lv 253:0 0 19G 0 lvm /
sr0 11:0 1 1.2G 0 rom
rbd0 252:0 0 5G 0 disk
3.1.6 格式化磁盤并掛載
#格式化磁盤蕊唐，xfs格式
root@ceshi:~# mkfs.xfs /dev/rbd0
meta-data=/dev/rbd0 isize=512 agcount=8, agsize=163840 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=1, sparse=1, rmapbt=0
= reflink=1
data = bsize=4096 blocks=1310720, imaxpct=25
= sunit=16 swidth=16 blks
naming =version 2 bsize=4096 ascii-ci=0, ftype=1
log =internal log bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=16 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
#掛載至/mnt
root@ceshi:~# mount /dev/rbd0 /mnt
#查看
root@ceshi:~# df -h
Filesystem Size Used Avail Use% Mounted on
udev 893M 0 893M 0% /dev
tmpfs 188M 1.4M 187M 1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv 19G 8.5G 9.2G 49% /
tmpfs 938M 0 938M 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 938M 0 938M 0% /sys/fs/cgroup
/dev/loop1 64M 64M 0 100% /snap/core20/2015
/dev/loop0 56M 56M 0 100% /snap/core18/2796
/dev/loop4 71M 71M 0 100% /snap/lxd/21029
/dev/loop2 64M 64M 0 100% /snap/core20/1974
/dev/loop3 56M 56M 0 100% /snap/core18/2790
/dev/loop5 92M 92M 0 100% /snap/lxd/24061
/dev/loop6 41M 41M 0 100% /snap/snapd/20290
/dev/loop7 41M 41M 0 100% /snap/snapd/20092
/dev/sda2 974M 310M 597M 35% /boot
overlay 19G 8.5G 9.2G 49% /var/lib/docker/overlay2/88274b473f877f4351d2100e95a62cf2bc183067334de81b8d247d97aa63d6ba/merged
tmpfs 188M 0 188M 0% /run/user/0
/dev/rbd0 5.0G 69M 5.0G 2% /mnt
#管理端驗(yàn)證鏡像狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls -p myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg1 3 GiB 2
myimg2 5 GiB 2 excl
3.1.7 驗(yàn)證ceph內(nèi)核模塊加載
掛載rbd之后系統(tǒng)內(nèi)核會(huì)自動(dòng)加載libceph模塊
root@ceshi:~# lsmod |grep ceph
libceph 327680 1 rbd
libcrc32c 16384 6 nf_conntrack,nf_nat,btrfs,xfs,raid456,libceph
root@ceshi:~# modinfo libceph
filename: /lib/modules/5.4.0-166-generic/kernel/net/ceph/libceph.ko
license: GPL
description: Ceph core library
author: Patience Warnick <patience@newdream.net>
author: Yehuda Sadeh <yehuda@hq.newdream.net>
author: Sage Weil <sage@newdream.net>
srcversion: 915EC0D99CBE44982F02F3B
depends: libcrc32c
retpoline: Y
intree: Y
name: libceph
vermagic: 5.4.0-166-generic SMP mod_unload modversions
sig_id: PKCS#7
signer: Build time autogenerated kernel key
sig_key: 12:DB:DC:2C:B2:2E:26:54:C5:B7:45:E4:C4:1F:DA:3F:04:C4:46:C0
sig_hashalgo: sha512
3.1.8 設(shè)置開(kāi)機(jī)自動(dòng)掛載
#；需要提前配置開(kāi)機(jī)自啟相關(guān)服務(wù)
root@ceshi:~# cat /etc/systemd/system/rc-local.service
[Unit]
Description=/etc/rc.local Compatibility
Documentation=man:systemd-rc-local-generator(8)
ConditionFileIsExecutable=/etc/rc.local
After=network.target
[Service]
Type=forking
ExecStart=/etc/rc.local start
TimeoutSec=0
RemainAfterExit=yes
GuessMainPID=no
[Install]
WantedBy=multi-user.target
Alias=rc-local.service
root@ceshi:~# cat /etc/rc.local
#!/bin/bash
/usr/bin/rbd --id tom -p myrbd1 map myimg2
mount /dev/rbd0 /mnt
[root@ceph-client ~]#chmod a+x /etc/rc.local
重啟服務(wù)器
root@ceshi:/mnt# rbd showmapped
id pool namespace image snap device
0 myrbd1 myimg2 - /dev/rbd0
root@ceshi:/mnt# df -h
Filesystem Size Used Avail Use% Mounted on
udev 893M 0 893M 0% /dev
tmpfs 188M 1.4M 187M 1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv 19G 8.9G 8.8G 51% /
tmpfs 938M 0 938M 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 938M 0 938M 0% /sys/fs/cgroup
/dev/loop0 56M 56M 0 100% /snap/core18/2796
/dev/loop2 56M 56M 0 100% /snap/core18/2790
/dev/loop3 64M 64M 0 100% /snap/core20/2015
/dev/loop1 64M 64M 0 100% /snap/core20/1974
/dev/loop5 71M 71M 0 100% /snap/lxd/21029
/dev/loop4 41M 41M 0 100% /snap/snapd/20290
/dev/sda2 974M 310M 597M 35% /boot
/dev/loop6 92M 92M 0 100% /snap/lxd/24061
/dev/loop7 41M 41M 0 100% /snap/snapd/20092
/dev/rbd0 5.0G 126M 4.9G 3% /mnt
overlay 19G 8.9G 8.8G 51% /var/lib/docker/overlay2/88274b473f877f4351d2100e95a62cf2bc183067334de81b8d247d97aa63d6ba/merged
tmpfs 188M 0 188M 0% /run/user/0
3.1.9 卸載rbd鏡像
root@ceshi:~# umount /mnt
root@ceshi:~# rbd --user tom -p myrbd1 unmap myimg2
3.1.10 刪除rbd鏡像
鏡像刪除后數(shù)據(jù)也會(huì)被刪除而且是無(wú)法恢復(fù)烁设，因此在執(zhí)行刪除操作的時(shí)候要慎重
#刪除myrbd1存儲(chǔ)池的myimg1鏡像
cephadmin@ceph-deploy:~/ceph-cluster$ rbd rm --pool myrbd1 --image myimg1
Removing image: 100% complete...done.
#驗(yàn)證鏡像
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls -p myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg2 5 GiB 2
3.2 RBD存儲(chǔ)空間回收
刪除完成的數(shù)據(jù)只是標(biāo)記為已經(jīng)被刪除替梨，但是不會(huì)從塊存儲(chǔ)立即清空
3.2.1 集群狀態(tài)
root@ceshi:~# ceph --user tom df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 2.0 TiB 1.9 TiB 6.5 GiB 6.5 GiB 0.32
TOTAL 2.0 TiB 1.9 TiB 6.5 GiB 6.5 GiB 0.32
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 631 GiB
mypool 2 32 0 B 0 0 B 0 631 GiB
rbd-data1 3 32 0 B 0 0 B 0 631 GiB
myrbd1 4 32 268 MiB 81 804 MiB 0.04 631 GiB
3.2.2 創(chuàng)建數(shù)據(jù)
#創(chuàng)建200M的文件
root@ceshi:~# dd if=/dev/zero of=/mnt/ceph-test-file bs=1M count=200
200+0 records in
200+0 records out
209715200 bytes (210 MB, 200 MiB) copied, 1.68353 s, 125 MB/s
3.2.3 查看ceph
root@ceshi:~# ceph --user tom df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 2.0 TiB 1.9 TiB 6.5 GiB 6.5 GiB 0.32
TOTAL 2.0 TiB 1.9 TiB 6.5 GiB 6.5 GiB 0.32
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 631 GiB
mypool 2 32 0 B 0 0 B 0 631 GiB
rbd-data1 3 32 0 B 0 0 B 0 631 GiB
myrbd1 4 32 268 MiB 81 804 MiB 0.04 631 GiB
3.2.4 刪除數(shù)據(jù)后檢查
root@ceshi:~# rm -rf /mnt/ceph-test-file
刪除完成的數(shù)據(jù)只是標(biāo)記為已經(jīng)被刪除，但是不會(huì)從塊存儲(chǔ)立即清空装黑，因此在刪除完成后使用ceph df 查看并沒(méi)有回收空間
root@ceshi:~# df -h
Filesystem Size Used Avail Use% Mounted on
udev 893M 0 893M 0% /dev
tmpfs 188M 1.4M 187M 1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv 19G 8.9G 8.8G 51% /
tmpfs 938M 0 938M 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 938M 0 938M 0% /sys/fs/cgroup
/dev/loop0 56M 56M 0 100% /snap/core18/2796
/dev/loop2 56M 56M 0 100% /snap/core18/2790
/dev/loop3 64M 64M 0 100% /snap/core20/2015
/dev/loop1 64M 64M 0 100% /snap/core20/1974
/dev/loop5 71M 71M 0 100% /snap/lxd/21029
/dev/loop4 41M 41M 0 100% /snap/snapd/20290
/dev/sda2 974M 310M 597M 35% /boot
/dev/loop6 92M 92M 0 100% /snap/lxd/24061
/dev/loop7 41M 41M 0 100% /snap/snapd/20092
overlay 19G 8.9G 8.8G 51% /var/lib/docker/overlay2/88274b473f877f4351d2100e95a62cf2bc183067334de81b8d247d97aa63d6ba/merged
tmpfs 188M 0 188M 0% /run/user/0
/dev/rbd0 5.0G 126M 4.9G 3% /mnt #顯示已刪除
root@ceshi:~# ceph --user tom df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 2.0 TiB 1.9 TiB 6.5 GiB 6.5 GiB 0.32
TOTAL 2.0 TiB 1.9 TiB 6.5 GiB 6.5 GiB 0.32
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 631 GiB
mypool 2 32 0 B 0 0 B 0 631 GiB
rbd-data1 3 32 0 B 0 0 B 0 631 GiB
myrbd1 4 32 268 MiB 81 804 MiB 0.04 631 GiB
3.3 RBD鏡像空間動(dòng)態(tài)伸縮
3.3.1 擴(kuò)容
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls -p myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg2 5 GiB 2 excl
#調(diào)整鏡像至20G
cephadmin@ceph-deploy:~/ceph-cluster$ rbd resize --pool myrbd1 --image myimg2 --size 20G
Resizing image: 100% complete...done.
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls -p myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg2 20 GiB 2
3.3.2 縮容
#通常不建議縮容
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls -p myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg2 20 GiB 2
#縮容至15G
cephadmin@ceph-deploy:~/ceph-cluster$ rbd resize --pool myrbd1 --image myimg2 --size 15G --allow-shrink
Resizing image: 100% complete...done.
cephadmin@ceph-deploy:~/ceph-cluster$ rbd ls -p myrbd1 -l
NAME SIZE PARENT FMT PROT LOCK
myimg2 15 GiB 2
3.3.3 客戶端驗(yàn)證
#fdisk已識(shí)別到15G
root@ceshi:~# fdisk -l /dev/rbd0
Disk /dev/rbd0: 15 GiB, 16106127360 bytes, 31457280 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 65536 bytes / 65536 bytes
#系統(tǒng)還未識(shí)別到
root@ceshi:~# df -h
Filesystem Size Used Avail Use% Mounted on
udev 893M 0 893M 0% /dev
tmpfs 188M 1.4M 187M 1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv 19G 8.9G 8.8G 51% /
tmpfs 938M 0 938M 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 938M 0 938M 0% /sys/fs/cgroup
/dev/loop0 56M 56M 0 100% /snap/core18/2796
/dev/loop2 56M 56M 0 100% /snap/core18/2790
/dev/loop3 64M 64M 0 100% /snap/core20/2015
/dev/loop1 64M 64M 0 100% /snap/core20/1974
/dev/loop5 71M 71M 0 100% /snap/lxd/21029
/dev/loop4 41M 41M 0 100% /snap/snapd/20290
/dev/sda2 974M 310M 597M 35% /boot
/dev/loop6 92M 92M 0 100% /snap/lxd/24061
/dev/loop7 41M 41M 0 100% /snap/snapd/20092
overlay 19G 8.9G 8.8G 51% /var/lib/docker/overlay2/88274b473f877f4351d2100e95a62cf2bc183067334de81b8d247d97aa63d6ba/merged
tmpfs 188M 0 188M 0% /run/user/0
/dev/rbd0 5.0G 126M 4.9G 3% /mnt
3.3.4 手動(dòng)執(zhí)行更新
如果是ext{2,3,4}文件系統(tǒng)的話副瀑，可以用resize2fs 命令來(lái)更新。
resize2fs /dev/rbd0
如果是xfs文件系統(tǒng)的話恋谭，用xfs_growfs更新
xfs_growfs /dev/rbd0
手動(dòng)執(zhí)行更新
root@ceshi:~# xfs_growfs /dev/rbd0
meta-data=/dev/rbd0 isize=512 agcount=8, agsize=163840 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=1, sparse=1, rmapbt=0
= reflink=1
data = bsize=4096 blocks=1310720, imaxpct=25
= sunit=16 swidth=16 blks
naming =version 2 bsize=4096 ascii-ci=0, ftype=1
log =internal log bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=16 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
data blocks changed from 1310720 to 3932160
#再次查看空間糠睡，已重新識(shí)別為15G
root@ceshi:~# df -h
Filesystem Size Used Avail Use% Mounted on
udev 893M 0 893M 0% /dev
tmpfs 188M 1.4M 187M 1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv 19G 8.9G 8.8G 51% /
tmpfs 938M 0 938M 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 938M 0 938M 0% /sys/fs/cgroup
/dev/loop0 56M 56M 0 100% /snap/core18/2796
/dev/loop2 56M 56M 0 100% /snap/core18/2790
/dev/loop3 64M 64M 0 100% /snap/core20/2015
/dev/loop1 64M 64M 0 100% /snap/core20/1974
/dev/loop5 71M 71M 0 100% /snap/lxd/21029
/dev/loop4 41M 41M 0 100% /snap/snapd/20290
/dev/sda2 974M 310M 597M 35% /boot
/dev/loop6 92M 92M 0 100% /snap/lxd/24061
/dev/loop7 41M 41M 0 100% /snap/snapd/20092
overlay 19G 8.9G 8.8G 51% /var/lib/docker/overlay2/88274b473f877f4351d2100e95a62cf2bc183067334de81b8d247d97aa63d6ba/merged
tmpfs 188M 0 188M 0% /run/user/0
/dev/rbd0 15G 198M 15G 2% /mnt
4.熟練通過(guò)命令管理ceph集群
4.1 ceph管理命令
- 只顯示存儲(chǔ)池
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool ls
device_health_metrics
mypool
myrbd1
- 列出存儲(chǔ)池并顯示id
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd lspools
1 device_health_metrics
2 mypool
4 myrbd1
- 查看pg狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph pg stat
65 pgs: 65 active+clean; 143 MiB data, 5.9 GiB used, 1.9 TiB / 2.0 TiB avail
- 查看指定pool或所有pool的狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool stats myrbd1
pool myrbd1 id 4
nothing is going on
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool stats
pool device_health_metrics id 1
nothing is going on
pool mypool id 2
nothing is going on
pool myrbd1 id 4
nothing is going on
- 查看集群存儲(chǔ)狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 2.0 TiB 1.9 TiB 5.9 GiB 5.9 GiB 0.30
TOTAL 2.0 TiB 1.9 TiB 5.9 GiB 5.9 GiB 0.30
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 631 GiB
mypool 2 32 0 B 0 0 B 0 631 GiB
myrbd1 4 32 68 MiB 96 204 MiB 0.01 631 GiB
- 查看集群存儲(chǔ)狀態(tài)詳情
cephadmin@ceph-deploy:~/ceph-cluster$ ceph df detail
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 2.0 TiB 1.9 TiB 5.9 GiB 5.9 GiB 0.30
TOTAL 2.0 TiB 1.9 TiB 5.9 GiB 5.9 GiB 0.30
--- POOLS ---
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 631 GiB N/A N/A N/A 0 B 0 B
mypool 2 32 0 B 0 B 0 B 0 0 B 0 B 0 B 0 631 GiB N/A N/A N/A 0 B 0 B
myrbd1 4 32 68 MiB 68 MiB 0 B 96 204 MiB 204 MiB 0 B 0.01 631 GiB N/A N/A N/A 0 B 0 B
- 查看osd狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd stat
20 osds: 20 up (since 26h), 20 in (since 8d); epoch: e146
- 顯示osd底層詳細(xì)信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd dump
epoch 146
fsid 3586e7d1-9315-44e5-85bd-6bd3787ce574
created 2023-10-26T03:38:29.325492+0000
modified 2023-11-04T03:36:18.790417+0000
flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit
crush_version 41
full_ratio 0.95
backfillfull_ratio 0.9
nearfull_ratio 0.85
require_min_compat_client luminous
min_compat_client luminous
require_osd_release pacific
stretch_mode_enabled false
pool 1 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 18 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr_devicehealth
pool 2 'mypool' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 113 flags hashpspool stripe_width 0
pool 4 'myrbd1' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 137 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd
max_osd 20
osd.0 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [5,116) [v2:172.20.20.226:6800/49905,v1:172.20.20.226:6801/49905] [v2:192.168.20.226:6809/1049905,v1:192.168.20.226:6804/1049905] exists,up ec6ffdcb-14ba-4cbd-afd4-35a302099686
osd.1 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [10,116) [v2:172.20.20.226:6804/51838,v1:172.20.20.226:6805/51838] [v2:192.168.20.226:6812/1051838,v1:192.168.20.226:6813/1051838] exists,up 2cea6bcd-f4e4-49d2-a98f-209042295165
osd.2 up in weight 1 up_from 118 up_thru 118 down_at 114 last_clean_interval [15,116) [v2:172.20.20.226:6808/53796,v1:172.20.20.226:6809/53796] [v2:192.168.20.226:6816/1053796,v1:192.168.20.226:6800/1053796] exists,up 020f8c18-e246-4126-8e4a-95a80f793c3f
osd.3 up in weight 1 up_from 118 up_thru 133 down_at 114 last_clean_interval [21,116) [v2:172.20.20.226:6812/55767,v1:172.20.20.226:6813/55767] [v2:192.168.20.226:6805/1055767,v1:192.168.20.226:6817/1055767] exists,up ef3c1b65-a665-42b3-ad67-b716723f3b9e
osd.4 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [26,116) [v2:172.20.20.226:6816/57728,v1:172.20.20.226:6817/57728] [v2:192.168.20.226:6808/1057728,v1:192.168.20.226:6801/1057728] exists,up 85d1abba-3f71-41b9-bfc4-5e3ce2edbfa3
osd.5 up in weight 1 up_from 118 up_thru 133 down_at 114 last_clean_interval [31,116) [v2:172.20.20.227:6800/94225,v1:172.20.20.227:6801/94225] [v2:192.168.20.227:6805/1094225,v1:192.168.20.227:6821/1094225] exists,up 9263d7d1-be58-42c5-ba04-57a1495b15b0
osd.6 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [37,117) [v2:172.20.20.227:6804/96278,v1:172.20.20.227:6805/96278] [v2:192.168.20.227:6812/1096278,v1:192.168.20.227:6813/1096278] exists,up 96f545ba-00d5-45ce-a243-08b6d15ae043
osd.7 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [43,117) [v2:172.20.20.227:6808/98240,v1:172.20.20.227:6809/98240] [v2:192.168.20.227:6800/1098240,v1:192.168.20.227:6801/1098240] exists,up 3e0dc4e1-6965-49fd-8323-322f139cb64b
osd.8 up in weight 1 up_from 118 up_thru 133 down_at 114 last_clean_interval [49,116) [v2:172.20.20.227:6812/100199,v1:172.20.20.227:6813/100199] [v2:192.168.20.227:6820/1100199,v1:192.168.20.227:6804/1100199] exists,up 9f18842e-828b-4661-86aa-9612a40de88d
osd.9 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [55,116) [v2:172.20.20.227:6816/102173,v1:172.20.20.227:6817/102173] [v2:192.168.20.227:6808/1102173,v1:192.168.20.227:6809/1102173] exists,up c0a0860e-4a96-42c9-ab8a-c145936d17f7
osd.10 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [60,116) [v2:172.20.20.228:6800/51897,v1:172.20.20.228:6801/51897] [v2:192.168.20.228:6804/1051897,v1:192.168.20.228:6808/1051897] exists,up 211e3030-2d14-4ec5-a46f-f77c61710a60
osd.11 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [66,116) [v2:172.20.20.228:6804/53921,v1:172.20.20.228:6805/53921] [v2:192.168.20.228:6801/1053921,v1:192.168.20.228:6812/1053921] exists,up 6b112c88-408c-4cd6-a55a-cf8888e65cb7
osd.12 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [71,116) [v2:172.20.20.228:6808/55905,v1:172.20.20.228:6809/55905] [v2:192.168.20.228:6805/1055905,v1:192.168.20.228:6800/1055905] exists,up 3296ef45-5070-45c7-9ed4-da6c4979d6f4
osd.13 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [77,116) [v2:172.20.20.228:6812/57830,v1:172.20.20.228:6813/57830] [v2:192.168.20.228:6816/1057830,v1:192.168.20.228:6817/1057830] exists,up 4366d617-e8a7-463d-830a-4937634a04d2
osd.14 up in weight 1 up_from 118 up_thru 129 down_at 114 last_clean_interval [82,116) [v2:172.20.20.228:6816/59778,v1:172.20.20.228:6817/59778] [v2:192.168.20.228:6809/1059778,v1:192.168.20.228:6813/1059778] exists,up fce1f4cc-b628-468e-9bd6-eb450d803a2d
osd.15 up in weight 1 up_from 87 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.20.20.229:6800/94054,v1:172.20.20.229:6801/94054] [v2:192.168.20.229:6800/94054,v1:192.168.20.229:6801/94054] exists,up a46e0031-b615-454d-ae47-bd5435dbd094
osd.16 up in weight 1 up_from 93 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.20.20.229:6804/96034,v1:172.20.20.229:6805/96034] [v2:192.168.20.229:6804/96034,v1:192.168.20.229:6805/96034] exists,up b58a7e96-e339-4e51-b06f-f518ef28765a
osd.17 up in weight 1 up_from 98 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.20.20.229:6808/97996,v1:172.20.20.229:6809/97996] [v2:192.168.20.229:6808/97996,v1:192.168.20.229:6809/97996] exists,up a9b0a97e-119e-4036-a783-166d63e13f78
osd.18 up in weight 1 up_from 104 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.20.20.229:6812/99905,v1:172.20.20.229:6813/99905] [v2:192.168.20.229:6812/99905,v1:192.168.20.229:6813/99905] exists,up 2fc94a7c-3a66-4861-bc5e-1d29b81ef8e1
osd.19 up in weight 1 up_from 109 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.20.20.229:6816/101915,v1:172.20.20.229:6817/101915] [v2:192.168.20.229:6816/101915,v1:192.168.20.229:6817/101915] exists,up 29294160-d10b-4908-a172-e6146c591026
pg_upmap_items 4.e [1,0]
pg_upmap_items 4.13 [7,5]
pg_upmap_items 4.17 [7,5]
- 顯示osd和節(jié)點(diǎn)對(duì)應(yīng)關(guān)系
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 1.95374 root default
-3 0.48843 host ceph-node1
0 hdd 0.09769 osd.0 up 1.00000 1.00000
1 hdd 0.09769 osd.1 up 1.00000 1.00000
2 hdd 0.09769 osd.2 up 1.00000 1.00000
3 hdd 0.09769 osd.3 up 1.00000 1.00000
4 hdd 0.09769 osd.4 up 1.00000 1.00000
-5 0.48843 host ceph-node2
5 hdd 0.09769 osd.5 up 1.00000 1.00000
6 hdd 0.09769 osd.6 up 1.00000 1.00000
7 hdd 0.09769 osd.7 up 1.00000 1.00000
8 hdd 0.09769 osd.8 up 1.00000 1.00000
9 hdd 0.09769 osd.9 up 1.00000 1.00000
-7 0.48843 host ceph-node3
10 hdd 0.09769 osd.10 up 1.00000 1.00000
11 hdd 0.09769 osd.11 up 1.00000 1.00000
12 hdd 0.09769 osd.12 up 1.00000 1.00000
13 hdd 0.09769 osd.13 up 1.00000 1.00000
14 hdd 0.09769 osd.14 up 1.00000 1.00000
-9 0.48843 host ceph-node4
15 hdd 0.09769 osd.15 up 1.00000 1.00000
16 hdd 0.09769 osd.16 up 1.00000 1.00000
17 hdd 0.09769 osd.17 up 1.00000 1.00000
18 hdd 0.09769 osd.18 up 1.00000 1.00000
19 hdd 0.09769 osd.19 up 1.00000 1.00000
到osd對(duì)應(yīng)的node節(jié)點(diǎn)查看與osd對(duì)應(yīng)的硬盤
root@ceph-node2:~# ll /var/lib/ceph/osd/ceph-6/block
lrwxrwxrwx 1 ceph ceph 93 Oct 26 06:09 /var/lib/ceph/osd/ceph-6/block -> /dev/ceph-c607f041-ede5-43f8-b0f5-e3f469e85aae/osd-block-96f545ba-00d5-45ce-a243-08b6d15ae043
root@ceph-node2:~# lsblk -f|grep -B1 ceph
sdb LVM2_member yJ74yq-ZGsY-sX9Z-ugQu-W1hh-oahf-xU5vws
└─ceph--fd63e9ce--8044--414d--b243--1589c826a29e-osd--block--9263d7d1--be58--42c5--ba04--57a1495b15b0 ceph_bluestore
sdc LVM2_member vNpNT8-5p5V-QSur-tR2u-aNWf-gIUR-7aHR2H
└─ceph--c607f041--ede5--43f8--b0f5--e3f469e85aae-osd--block--96f545ba--00d5--45ce--a243--08b6d15ae043 ceph_bluestore
sdd LVM2_member hhPjwX-sNdr-jq10-RgJn-P4Jt-HayT-89HOxe
└─ceph--bfce6bba--cf00--43d4--8e03--3ec3feee7084-osd--block--3e0dc4e1--6965--49fd--8323--322f139cb64b ceph_bluestore
sde LVM2_member dw8Cxq-cnke-zdKL-UM3T-1Tl0-v41g-cy5p6n
└─ceph--54e692a9--a59e--497d--b11f--0eb2b8f88e38-osd--block--9f18842e--828b--4661--86aa--9612a40de88d ceph_bluestore
sdf LVM2_member ugfKrv-OxwY-8j85-k0VT-7yeg-ciBZ-9MQl5c
└─ceph--1e1c1483--5594--44e7--874e--25582b2cb413-osd--block--c0a0860e--4a96--42c9--ab8a--c145936d17f7 ceph_bluestore
image.png
- 顯示osd存儲(chǔ)信息和節(jié)點(diǎn)對(duì)應(yīng)關(guān)系
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
-1 1.95374 - 2.0 TiB 5.9 GiB 241 MiB 0 B 5.7 GiB 1.9 TiB 0.30 1.00 - root default
-3 0.48843 - 500 GiB 1.5 GiB 49 MiB 0 B 1.4 GiB 499 GiB 0.29 0.99 - host ceph-node1
0 hdd 0.09769 1.00000 100 GiB 294 MiB 4.0 MiB 0 B 290 MiB 100 GiB 0.29 0.97 6 up osd.0
1 hdd 0.09769 1.00000 100 GiB 302 MiB 12 MiB 0 B 290 MiB 100 GiB 0.29 1.00 15 up osd.1
2 hdd 0.09769 1.00000 100 GiB 296 MiB 5.9 MiB 0 B 290 MiB 100 GiB 0.29 0.98 6 up osd.2
3 hdd 0.09769 1.00000 100 GiB 312 MiB 22 MiB 0 B 290 MiB 100 GiB 0.30 1.03 14 up osd.3
4 hdd 0.09769 1.00000 100 GiB 296 MiB 5.9 MiB 0 B 290 MiB 100 GiB 0.29 0.98 6 up osd.4
-5 0.48843 - 500 GiB 1.5 GiB 68 MiB 0 B 1.4 GiB 498 GiB 0.30 1.01 - host ceph-node2
5 hdd 0.09769 1.00000 100 GiB 304 MiB 14 MiB 0 B 290 MiB 100 GiB 0.30 1.01 5 up osd.5
6 hdd 0.09769 1.00000 100 GiB 296 MiB 5.9 MiB 0 B 290 MiB 100 GiB 0.29 0.98 10 up osd.6
7 hdd 0.09769 1.00000 100 GiB 322 MiB 28 MiB 0 B 294 MiB 100 GiB 0.31 1.07 14 up osd.7
8 hdd 0.09769 1.00000 100 GiB 292 MiB 1.9 MiB 0 B 290 MiB 100 GiB 0.29 0.97 9 up osd.8
9 hdd 0.09769 1.00000 100 GiB 308 MiB 18 MiB 0 B 290 MiB 100 GiB 0.30 1.02 9 up osd.9
-7 0.48843 - 500 GiB 1.5 GiB 71 MiB 0 B 1.4 GiB 498 GiB 0.30 1.01 - host ceph-node3
10 hdd 0.09769 1.00000 100 GiB 296 MiB 5.9 MiB 0 B 290 MiB 100 GiB 0.29 0.98 5 up osd.10
11 hdd 0.09769 1.00000 100 GiB 292 MiB 1.8 MiB 0 B 290 MiB 100 GiB 0.29 0.97 5 up osd.11
12 hdd 0.09769 1.00000 100 GiB 318 MiB 28 MiB 0 B 290 MiB 100 GiB 0.31 1.05 14 up osd.12
13 hdd 0.09769 1.00000 100 GiB 312 MiB 22 MiB 0 B 290 MiB 100 GiB 0.30 1.03 16 up osd.13
14 hdd 0.09769 1.00000 100 GiB 304 MiB 14 MiB 0 B 290 MiB 100 GiB 0.30 1.01 11 up osd.14
-9 0.48843 - 500 GiB 1.5 GiB 53 MiB 0 B 1.4 GiB 499 GiB 0.29 0.99 - host ceph-node4
15 hdd 0.09769 1.00000 100 GiB 318 MiB 27 MiB 0 B 290 MiB 100 GiB 0.31 1.05 11 up osd.15
16 hdd 0.09769 1.00000 100 GiB 292 MiB 1.9 MiB 0 B 290 MiB 100 GiB 0.29 0.97 4 up osd.16
17 hdd 0.09769 1.00000 100 GiB 292 MiB 1.9 MiB 0 B 290 MiB 100 GiB 0.29 0.97 13 up osd.17
18 hdd 0.09769 1.00000 100 GiB 300 MiB 10 MiB 0 B 290 MiB 100 GiB 0.29 0.99 12 up osd.18
19 hdd 0.09769 1.00000 100 GiB 302 MiB 12 MiB 0 B 290 MiB 100 GiB 0.30 1.00 10 up osd.19
TOTAL 2.0 TiB 5.9 GiB 241 MiB 0 B 5.7 GiB 1.9 TiB 0.30
MIN/MAX VAR: 0.97/1.07 STDDEV: 0.01
- 查看mon節(jié)點(diǎn)狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mon stat
e3: 3 mons at {ceph-mon1=[v2:172.20.20.221:3300/0,v1:172.20.20.221:6789/0],ceph-mon2=[v2:172.20.20.222:3300/0,v1:172.20.20.222:6789/0],ceph-mon3=[v2:172.20.20.223:3300/0,v1:172.20.20.223:6789/0]} removed_ranks: {}, election epoch 18, leader 0 ceph-mon1, quorum 0,1,2 ceph-mon1,ceph-mon2,ceph-mon3
- 查看mon節(jié)點(diǎn)的dump信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mon dump
epoch 3
fsid 3586e7d1-9315-44e5-85bd-6bd3787ce574
last_changed 2023-11-03T02:15:29.548725+0000
created 2023-10-26T03:38:28.654596+0000
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:172.20.20.221:3300/0,v1:172.20.20.221:6789/0] mon.ceph-mon1
1: [v2:172.20.20.222:3300/0,v1:172.20.20.222:6789/0] mon.ceph-mon2
2: [v2:172.20.20.223:3300/0,v1:172.20.20.223:6789/0] mon.ceph-mon3
dumped monmap epoch 3
4.2 ceph集群維護(hù)
4.2.1 通過(guò)套接字進(jìn)行單機(jī)管理
在ceph的節(jié)點(diǎn)上使用socket管理只針對(duì)ceph的節(jié)點(diǎn)單機(jī)管理并不會(huì)對(duì)所有節(jié)點(diǎn)生效
#node節(jié)點(diǎn)
root@ceph-node1:~# ll /var/run/ceph/
total 0
drwxrwx--- 2 ceph ceph 140 Oct 26 06:08 ./
drwxr-xr-x 31 root root 1000 Nov 4 03:59 ../
srwxr-xr-x 1 ceph ceph 0 Oct 26 06:07 ceph-osd.0.asok=
srwxr-xr-x 1 ceph ceph 0 Oct 26 06:08 ceph-osd.1.asok=
srwxr-xr-x 1 ceph ceph 0 Oct 26 06:08 ceph-osd.2.asok=
srwxr-xr-x 1 ceph ceph 0 Oct 26 06:08 ceph-osd.3.asok=
srwxr-xr-x 1 ceph ceph 0 Oct 26 06:08 ceph-osd.4.asok=
#mon節(jié)點(diǎn)
root@ceph-mon1:~# ll /var/run/ceph/
total 0
drwxrwx--- 2 ceph ceph 60 Oct 26 03:38 ./
drwxr-xr-x 31 root root 1000 Nov 4 04:01 ../
srwxr-xr-x 1 ceph ceph 0 Oct 26 03:38 ceph-mon.ceph-mon1.asok=
注意：
在 node 節(jié)點(diǎn)或者 mon 節(jié)點(diǎn)通過(guò) ceph 命令進(jìn)行單機(jī)管理本機(jī)的 mon 或者 osd 服務(wù)
要先將 admin 認(rèn)證文件同步到 mon 或者 node 節(jié)點(diǎn)
#在mon節(jié)點(diǎn)查看mon狀態(tài)
root@ceph-mon1:~# ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok mon_status
#查看配置信息
root@ceph-mon1:~# ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok config show
4.2.2 ceph集群的停止或重啟
OSD的維護(hù)
重啟之前，要提前設(shè)置 ceph 集群不要將 OSD 標(biāo)記為 out疚颊，以及將backfill和recovery設(shè)置為no狈孔，避免 node 節(jié)點(diǎn)關(guān)閉服務(wù)后osd被踢出 ceph 集群外，以及存儲(chǔ)池進(jìn)行修復(fù)數(shù)據(jù)材义，等待節(jié)點(diǎn)維護(hù)完成后均抽，再將所有標(biāo)記取消設(shè)置。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd set noout
noout is set
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd set norecover
norecover is set
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd set nobackfill
nobackfill is set
#查看
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: 3586e7d1-9315-44e5-85bd-6bd3787ce574
health: HEALTH_WARN
noout,nobackfill,norecover flag(s) set
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 117m)
mgr: ceph-mgr1(active, since 9d), standbys: ceph-mgr2
osd: 20 osds: 20 up (since 27h), 20 in (since 8d)
flags noout,nobackfill,norecover
data:
pools: 3 pools, 65 pgs
objects: 96 objects, 143 MiB
usage: 5.9 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 65 active+clean
當(dāng)ceph的節(jié)點(diǎn)恢復(fù)時(shí)其掂，就是用unset取消標(biāo)記油挥，使集群的osd開(kāi)始重新服務(wù)，并開(kāi)始修復(fù)數(shù)據(jù)。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd unset noout
noout is unset
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd unset nobackfill
nobackfill is unset
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd unset norecover
norecover is unset
ceph集群服務(wù)停機(jī)關(guān)閉順序
- 確保ceph集群當(dāng)前為noout喘漏、nobackfill护蝶、norecover狀態(tài)
- 關(guān)閉存儲(chǔ)客戶端停止讀寫數(shù)據(jù)
- 如果使用了 RGW华烟，關(guān)閉 RGW
- 關(guān)閉 cephfs 元數(shù)據(jù)服務(wù)
- 關(guān)閉 ceph OSD
- 關(guān)閉 ceph manager
- 關(guān)閉 ceph monitor
ceph集群?jiǎn)?dòng)順序
- 啟動(dòng) ceph monitor
- 啟動(dòng) ceph manager
- 啟動(dòng) ceph OSD
- 關(guān)閉 cephfs 元數(shù)據(jù)服務(wù)
- 啟動(dòng) RGW
- 啟動(dòng)存儲(chǔ)客戶端
- 啟動(dòng)服務(wù)后取消 noout-->ceph osd unset noout
4.2.3 添加節(jié)點(diǎn)服務(wù)器
- 添加ceph倉(cāng)庫(kù)源
- 安裝ceph服務(wù)
#node節(jié)點(diǎn)安裝執(zhí)行
apt install python-pip
#在部署節(jié)點(diǎn)執(zhí)行
ceph-deploy install --release pacific {ceph-nodeN}
- 擦除磁盤
ceph-deploy disk zap {ceph-nodeN} {/dev/sdX}
- 添加osd
ceph-deploy osd create {ceph-nodeN} --data {/dev/sdX}
4.2.4 刪除OSD或服務(wù)器
把故障OSD從ceph集群刪除
- 把osd踢出集群
ceph osd out osd.{id}
- 等一段時(shí)間翩迈，等ceph數(shù)據(jù)修復(fù)
- 進(jìn)入對(duì)應(yīng)node節(jié)點(diǎn)，停止osd.{id}進(jìn)程
systemctl stop ceph-osd@{id}.service
- 刪除osd
ceph osd rm {id}
4.2.5 刪除服務(wù)器
刪除服務(wù)器之前要把該服務(wù)器上所有OSD先停止并從ceph集群移除
- 把osd踢出集群
- 等一段時(shí)間
- 進(jìn)入對(duì)應(yīng)node節(jié)點(diǎn)盔夜，停止osd.{id}進(jìn)程
- 刪除osd
- 重復(fù)上述步驟负饲，刪除該node節(jié)點(diǎn)上所有osd
- osd全部操作完成后下線主機(jī)
- 從crush刪除ceph-nodeN節(jié)點(diǎn)
ceph osd crush rm ceph ceph-nodeN
5.熟悉pg的常見(jiàn)狀態(tài)
PG的常見(jiàn)狀態(tài)如下：
-
peering
正在同步狀態(tài)，同一個(gè)PG中的OSD需要將準(zhǔn)備數(shù)據(jù)同步一致喂链，而peering（對(duì)等）就是OSD同步過(guò)程中的狀態(tài)返十。 -
activating
Peering 已經(jīng)完成，PG正在等待所有PG實(shí)例同步Peering的結(jié)果(Info椭微、Log等) -
clean
干凈態(tài)洞坑，PG當(dāng)前不存在待修復(fù)的對(duì)象，并且大小等于存儲(chǔ)池的副本數(shù)蝇率，即PG的活動(dòng)集(Acting Set)和上行集(Up Set)為同一組OSD且內(nèi)容一致迟杂。
活動(dòng)集(Acting Set)：由PG當(dāng)前主的OSD和其余處于活動(dòng)狀態(tài)的備用OSD組成，當(dāng)前PG內(nèi)的OSD負(fù)責(zé)處理用戶的讀寫請(qǐng)求本慕。
上行集(Up Set)：在某一個(gè)OSD故障時(shí)排拷，需要將故障的OSD更換為可用的OSD，并主PG內(nèi)部的主OSD同步數(shù)據(jù)到新的OSD上锅尘，例如PG內(nèi)有OSD1监氢、OSD2、OSD3藤违，當(dāng)OSD3故障后需要用OSD4替換OSD3浪腐，那么OSD1、OSD2顿乒、OSD3就是上行集议街，替換后OSD1,OSD2、OSD4就是活動(dòng)集淆游，OSD替換完成后活動(dòng)集最終要替換上行集傍睹。 -
active
就緒狀態(tài)或活躍狀態(tài)，Active表示主OSD和備OSD處于正常工作狀態(tài)犹菱，此時(shí)的PG可以正常處理來(lái)自客戶端的讀寫請(qǐng)求拾稳，正常的PG默認(rèn)就是Active+Clean狀態(tài)。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph pg stat
65 pgs: 65 active+clean; 143 MiB data, 5.9 GiB used, 1.9 TiB / 2.0 TiB avail
-
degraded
降級(jí)狀態(tài)腊脱，該狀態(tài)出現(xiàn)于OSD被標(biāo)記為down以后访得，那么其他映射到此OSD的PG都會(huì)轉(zhuǎn)換到降級(jí)狀態(tài)。
如果此OSD還能重新啟動(dòng)完成并完成Peering操作后,那么使用此OSD的PG將重新恢復(fù)為clean狀態(tài)。
如果此OSD被標(biāo)記為down的時(shí)間超過(guò)5分鐘還沒(méi)有修復(fù)悍抑，那么此OSD將會(huì)被ceph踢出集群鳄炉，然后ceph會(huì)對(duì)被降級(jí)的PG啟動(dòng)恢復(fù)操作，直到所有由于此OSD而被降級(jí)的PG重新恢復(fù)為clean狀態(tài)搜骡。
恢復(fù)數(shù)據(jù)會(huì)從PG內(nèi)的主OSD恢復(fù)拂盯，如果是主OSD故障，那么會(huì)在剩下的兩個(gè)備用OSD重新選擇一個(gè)作為主OSD记靡。 -
stale
過(guò)期狀態(tài)谈竿，正常情況下每個(gè)主OSD都要周期性的向RADOS集群中的監(jiān)視器(Mon)報(bào)告其作為主OSD所持有的所有PG的最新統(tǒng)計(jì)數(shù)據(jù)，因任何原因?qū)е履硞€(gè)OSD無(wú)法正常向監(jiān)視器發(fā)送匯報(bào)信息的摸吠、或者由其他OSD報(bào)告某個(gè)OSD已經(jīng)down 的時(shí)候空凸，則所有以此OSD為主PG則會(huì)立即被標(biāo)記為stale狀態(tài)，即他們的主OSD已經(jīng)不是最新的數(shù)據(jù)了寸痢，如果是備份的OSD發(fā)送down的時(shí)候呀洲，則ceph會(huì)執(zhí)行修復(fù)而不會(huì)觸發(fā)PG狀態(tài)轉(zhuǎn)換為stale狀態(tài)。 -
undersized
小于正常狀態(tài)啼止，PG當(dāng)前副本數(shù)小于其存儲(chǔ)池定義的值的時(shí)候道逗，PG會(huì)轉(zhuǎn)換為undersized狀態(tài)，比如兩個(gè)備份OSD都down了族壳，那么此時(shí)PG中就只有一個(gè)主OSD了憔辫，不符合ceph最少要求一個(gè)主OSD加一個(gè)備OSD的要求，那么就會(huì)導(dǎo)致使用此OSD的PG轉(zhuǎn)換為undersized 狀態(tài)仿荆，直到添加備份OSD添加完成贰您，或者修復(fù)完成。 -
scrubbing
scrub是ceph對(duì)數(shù)據(jù)的清洗狀態(tài)拢操，用來(lái)保證數(shù)據(jù)完整性的機(jī)制锦亦，Ceph 的OSD定期啟動(dòng)scrub線程來(lái)掃描部分對(duì)象，通過(guò)與其他副本比對(duì)來(lái)發(fā)現(xiàn)是否一致令境，如果存在不一致杠园，拋出異常提示用戶手動(dòng)解決, scrub 以PG為單位，對(duì)于每一個(gè)pg舔庶， ceph分析該pg下所有的object抛蚁，產(chǎn)生一個(gè)類似于元數(shù)據(jù)信息摘要的數(shù)據(jù)結(jié)構(gòu),如對(duì)象大小,屬性等,叫scrubmap,比較主與副scrubmap，來(lái)保證是不是有object丟失或者不匹配惕橙，掃描分為輕量級(jí)掃描和深度掃描瞧甩，輕量級(jí)掃描也叫做light scrubs或者shallow scrubs或者simply scrubs即輕量級(jí)掃描.
Light scrub(daily)比較object size 和屬性, deep scrub (weekly)讀取數(shù)據(jù)部分并通過(guò)checksum(CRC32算法)對(duì)比和數(shù)據(jù)的一致性,深度掃描過(guò)程中的PG會(huì)處于scrubbing+deep狀態(tài). -
recovering
正在恢復(fù)態(tài)，集群正在執(zhí)行遷移或同步對(duì)象和他們的副本弥鹦，這可能是由于添加了一個(gè)新的OSD到集群中或者某個(gè)OSD宕掉后肚逸，PG可能會(huì)被CRUSH算法重新分配不同的OSD，而由于OSD更換導(dǎo)致PG發(fā)生內(nèi)部數(shù)據(jù)同步的過(guò)程中的PG會(huì)被標(biāo)記為Recovering. -
backfilling
正在后臺(tái)填充態(tài), backfill是recovery 的一種特殊場(chǎng)景，指peering完成后朦促，如果基于當(dāng)前權(quán)威日志無(wú)法對(duì)Up Set(上行集)當(dāng)中的某些PG實(shí)例實(shí)施增量同步(例如承載這些PG實(shí)例的OSD離線太久,或者是新的OSD加入集群導(dǎo)致的PG實(shí)例整體遷移)則通過(guò)完全拷貝當(dāng)前Primary所有對(duì)象的方式進(jìn)行全量同步膝晾，此過(guò)程中的PG會(huì)處于backfilling. -
backfill-toofull
某個(gè)需要被backfill的PG實(shí)例，其所在的OSD可用空間不足务冕，Backfill流程當(dāng)前被掛起時(shí)PG給的狀態(tài)血当。
6.掌握cephfs的部署和使用
6.1 cephfs介紹
ceph FS 即 ceph filesystem，可以實(shí)現(xiàn)文件系統(tǒng)共享功能(POSIX 標(biāo)準(zhǔn)), 客戶端通過(guò) ceph協(xié)議掛載并使用 ceph 集群作為數(shù)據(jù)存儲(chǔ)服務(wù)器洒疚，https://docs.ceph.com/en/quincy/cephfs/歹颓。
Ceph FS 需要運(yùn)行 Meta Data Services(MDS)服務(wù)坯屿，其守護(hù)進(jìn)程為 ceph-mds油湖，ceph-mds進(jìn)程管理與 cephFS 上存儲(chǔ)的文件相關(guān)的元數(shù)據(jù)，并協(xié)調(diào)對(duì) ceph 存儲(chǔ)集群的訪問(wèn)领跛。
在 linux 系統(tǒng)使用 ls 等操作查看某個(gè)目錄下的文件的時(shí)候乏德，會(huì)有保存在磁盤上的分區(qū)表記錄文件的名稱、創(chuàng)建日期吠昭、大小喊括、inode 及存儲(chǔ)位置等元數(shù)據(jù)信息，在 cephfs 由于數(shù)據(jù)是被打散為若干個(gè)離散的 object 進(jìn)行分布式存儲(chǔ)矢棚，因此并沒(méi)有統(tǒng)一保存文件的元數(shù)據(jù)郑什，而且將文件的元數(shù)據(jù)保存到一個(gè)單獨(dú)的存儲(chǔ)出 matedata pool，但是客戶端并不能直接訪問(wèn)matedata pool 中的元數(shù)據(jù)信息蒲肋，而是在讀寫數(shù)的時(shí)候有 MDS(matadata server)進(jìn)行處理蘑拯，讀數(shù)據(jù)的時(shí)候由 MDS從 matedata pool加載元數(shù)據(jù)然后緩存在內(nèi)存(用于后期快速響應(yīng)其它客戶端的請(qǐng)求)并返回給客戶端，寫數(shù)據(jù)的時(shí)候有 MDS 緩存在內(nèi)存并同步到 matedata pool兜粘。
image.png
cephfs 的 mds 的數(shù)據(jù)結(jié)構(gòu)類似于 linux 系統(tǒng)的根形目錄結(jié)構(gòu)及 nginx 中的緩存目錄分層一樣申窘。
image.png
6.2 cephfs部署
在指定的 ceph-mds 服務(wù)器部署 ceph-mds 服務(wù)，可以和其它服務(wù)器混用(如 ceph-mon孔轴、ceph-mgr)
#ubuntu
apt install ceph-mds
#centos
yum install ceph-mds
#進(jìn)入部署節(jié)點(diǎn)剃法，指定在ceph-mgr1節(jié)點(diǎn)安裝ceph-mds
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mds create ceph-mgr1
#驗(yàn)證
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
1 up:standby #當(dāng)前為備用狀態(tài)，需要分配 pool 才可以使用路鹰。
6.3 創(chuàng)建 CephFS metadata 和 data 存儲(chǔ)池
使用 CephFS 之前需要事先于集群中創(chuàng)建一個(gè)文件系統(tǒng)贷洲，并為其分別指定元數(shù)據(jù)和數(shù)據(jù)相關(guān)的存儲(chǔ)池。
創(chuàng)建名為 mycephfs 的文件系統(tǒng)晋柱，使用 cephfs-metadata 為 元數(shù)據(jù)存儲(chǔ)池优构，使用 cephfs-data 為數(shù)據(jù)存儲(chǔ)池
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-metadata 32 32
pool 'cephfs-metadata' created
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-data 64 64
pool 'cephfs-data' created
#查看ceph狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: 3586e7d1-9315-44e5-85bd-6bd3787ce574
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 2h)
mgr: ceph-mgr1(active, since 9d), standbys: ceph-mgr2
osd: 20 osds: 20 up (since 28h), 20 in (since 8d)
data:
pools: 5 pools, 161 pgs
objects: 96 objects, 143 MiB
usage: 5.9 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 161 active+clean
注意：在實(shí)際的生產(chǎn)使用中，cephfs數(shù)據(jù)存儲(chǔ)池存儲(chǔ)了幾十T的數(shù)據(jù)趣斤，那么元數(shù)據(jù)的存儲(chǔ)池大約占用幾個(gè)G空間俩块。
6.4 創(chuàng)建 cephFS
創(chuàng)建cephfs，指定fs的元數(shù)據(jù)池和fs數(shù)據(jù)池
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new share1 cephfs-metadata cephfs-data
#驗(yàn)證
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls
name: share1, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
#查看cephfs的狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status share1
share1 - 0 clients
======
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mgr1 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 631G
cephfs-data data 0 631G
MDS version: ceph version 16.2.14 (238ba602515df21ea7ffc75c88db29f9e5ef12c9) pacific (stable)
#查看ceph狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: 3586e7d1-9315-44e5-85bd-6bd3787ce574
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 2h)
mgr: ceph-mgr1(active, since 9d), standbys: ceph-mgr2
mds: 1/1 daemons up
osd: 20 osds: 20 up (since 28h), 20 in (since 8d)
data:
volumes: 1/1 healthy
pools: 5 pools, 161 pgs
objects: 118 objects, 143 MiB
usage: 5.9 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 161 active+clean
#查看mds節(jié)點(diǎn)狀態(tài)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
share1:1 {0=ceph-mgr1=up:active}
