1岖寞、環(huán)境準備

1.1 安裝系統(tǒng)

[root@node242 ~]# cat /etc/redhat-release
CentOS Linux release 7.2.1511 (Core)

1.2 關(guān)閉selinux

[root@node242 ~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
[root@node242 ~]# setenforce 0

1.3 配置yum源

[root@node242 ~]# curl -o /etc/yum.repos.d/openstack-mitaka-ceph-hammer-7.2-x86_64.repo http://172.30.120.146/repo.files/mitaka.x86.repo
[root@node242 ~]# yum clean all
[root@node242 ~]# yum makecache

1.4 時間同步

[root@node242 ~]# yum install ntp -y
[root@node242 ~]# ntpdate cn.pool.ntp.org
 1 Aug 09:46:23 ntpdate[23508]: step time server 202.108.6.95 offset 973.598833 sec

2、部署 Ceph 集群

2.1 安裝 ceph

[root@node242 ~]# yum install ceph -y

2.2 創(chuàng)建 ceph.conf 配置文件

[root@node242 ~]# uuidgen
346436e3-a7f4-4fd9-9694-e9830faec4d5

[root@node242 ~]# vim /etc/ceph/ceph.conf
[global]
fsid = 346436e3-a7f4-4fd9-9694-e9830faec4d5
mon initial members = node242
mon host = 192.168.100.242
public network = 192.168.100.0/24
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 1
osd pool default min size = 1
osd pool default pg num = 128
osd pool default pgp num = 128
osd crush chooseleaf type = 1
osd mkfs type = btrfs
mon clock drift allowed = 2
mon clock drift warn backoff = 30

[mon.node242]
host = node242
mon addr = 192.168.100.242:6789

2.3 部署配置 Ceph Monitors

[root@node242 ~]# mkdir ceph
[root@node242 ~]# cd ceph/
[root@node242 ceph]# ls

創(chuàng)建一個 ceph.mon.keyring

[root@node242 ceph]# ceph-authtool --create-keyring ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
creating ceph.mon.keyring

創(chuàng)建一個 ceph.client.admin.keyring

[root@node242 ceph]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'
creating /etc/ceph/ceph.client.admin.keyring

將ceph.client.admin.keyring導入到ceph.mon.keyring

[root@node242 ceph]# ceph-authtool ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
importing contents of /etc/ceph/ceph.client.admin.keyring into ceph.mon.keyring

創(chuàng)建一個Mon焦辅，名字叫${HOSTNAME}(此處可以直接寫主機名，如: node242)

[root@node242 ceph]# monmaptool --create --add ${HOSTNAME} 192.168.100.242 --fsid 346436e3-a7f4-4fd9-9694-e9830faec4d5 ./monmap
monmaptool: monmap file monmap
monmaptool: set fsid to 346436e3-a7f4-4fd9-9694-e9830faec4d5
monmaptool: writing epoch 0 to monmap (1 monitors)

創(chuàng)建存儲 monitor 數(shù)據(jù)的文件夾，文件夾里主要有keyring和store.db

[root@node242 ceph]# mkdir /var/lib/ceph/mon/ceph-${HOSTNAME}

使用 monmap 和 ceph.mon.keyring 初始化 ceph-mon 守護進程開啟需要的數(shù)據(jù)文件keyring和store.db

[root@node242 ceph]# ceph-mon --mkfs -i ${HOSTNAME} --monmap ./monmap --keyring ./ceph.mon.keyring
2019-07-31 19:27:03.659441 7f2f416b0880 -1 did not load config file, using default settings.
ceph-mon: created monfs at /var/lib/ceph/mon/ceph-node242 for mon.node242

在 monitor 數(shù)據(jù)目錄下創(chuàng)建 done 文件成翩，防止重新建立MON

[root@node242 ceph]# touch /var/lib/ceph/mon/ceph-${HOSTNAME}/done

啟動 ceph-mon

[root@node242 ceph]# /etc/init.d/ceph start mon.node242
=== mon.node242 ===
Starting Ceph mon.node242 on node242...
Running as unit run-10346.service.
Starting ceph-create-keys on node242...
[root@node242 ceph]# /etc/init.d/ceph status mon.node242
=== mon.node242 ===
mon.node242: running {"version":"0.94.5"}

查看集群狀態(tài)

[root@node242 ceph]# ceph -s
    cluster 346436e3-a7f4-4fd9-9694-e9830faec4d5
     health HEALTH_ERR
            64 pgs stuck inactive
            64 pgs stuck unclean
            no osds
     monmap e1: 1 mons at {node242=192.168.100.242:6789/0}
            election epoch 2, quorum 0 node242
     osdmap e1: 0 osds: 0 up, 0 in
      pgmap v2: 64 pgs, 1 pools, 0 bytes data, 0 objects
            0 kB used, 0 kB / 0 kB avail
                  64 creating
[root@node242 ceph]# ceph osd tree
ID WEIGHT TYPE NAME    UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1      0 root default
[root@node242 ~]# ceph osd pool ls detail
pool 0 'rbd' replicated size 1 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 flags hashpspool stripe_width 0

2.4 部署配置 Ceph OSD

查看osd節(jié)點磁盤

[root@node242 ~]# lsblk 
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vda             252:0    0   80G  0 disk 
├─vda1          252:1    0  500M  0 part /boot
└─vda2          252:2    0 79.5G  0 part 
  ├─centos-root 253:0    0   75G  0 lvm  /
  └─centos-swap 253:1    0  3.9G  0 lvm  [SWAP]
vdb             252:16   0    8G  0 disk 
vdc             252:32   0    8G  0 disk 
vdd             252:48   0    8G  0 disk

查看磁盤分區(qū)表

[root@node242 ~]# parted /dev/vdb print
Error: /dev/vdb: unrecognised disk label
Model: Virtio Block Device (virtblk)                                      
Disk /dev/vdb: 8590MB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:

把磁盤分區(qū)表設(shè)置為 gpt

[root@node242 ~]# parted /dev/vdb mktable gpt
Information: You may need to update /etc/fstab.

查看磁盤分區(qū)表

[root@node242 ~]# parted /dev/vdb print
Model: Virtio Block Device (virtblk)
Disk /dev/vdb: 8590MB
Sector size (logical/physical): 512B/512B
Partition Table: gpt
Disk Flags:

Number  Start  End  Size  File system  Name  Flags

創(chuàng)建分區(qū)1作為日志盤

[root@node242 ~]# parted /dev/vdb  mkpart primary xfs 0% 20%
Information: You may need to update /etc/fstab.

創(chuàng)建分區(qū)2作為數(shù)據(jù)盤

[root@node242 ~]# parted /dev/vdb  mkpart primary xfs 20% 100%
Information: You may need to update /etc/fstab.

查看磁盤分區(qū)

[root@node242 ~]# lsblk /dev/vdb
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vdb    252:16   0    8G  0 disk 
├─vdb1 252:17   0  1.6G  0 part 
└─vdb2 252:18   0  6.4G  0 part

創(chuàng)建 osd , 分配 osd id

[root@node242 ~]# ceph osd tree
ID WEIGHT TYPE NAME    UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1      0 root default
[root@node242 ~]# ceph osd create
0
[root@node242 ~]# ceph osd tree
ID WEIGHT  TYPE NAME        UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 2.00000 root default                                       
 0       0 osd.0               down        0          1.00000

創(chuàng)建存儲 osd 數(shù)據(jù)的文件夾氮惯，掛載 osd 數(shù)據(jù)盤

[root@node242 ~]# mkdir /var/lib/ceph/osd/ceph-0
[root@node242 ~]# mkfs.xfs -f /dev/vdb2 
meta-data=/dev/vdb2              isize=256    agcount=4, agsize=419392 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=0        finobt=0
data     =                       bsize=4096   blocks=1677568, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=0
log      =internal log           bsize=4096   blocks=2560, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0
[root@node242 ~]# mount /dev/vdb2 /var/lib/ceph/osd/ceph-0/

配置 osd 日志盤

[root@node242 ~]# ll /dev/disk/by-partuuid/
total 0
lrwxrwxrwx 1 root root 10 Aug  1 11:18 b0d06586-00e8-42fe-bdad-9b47abee374e -> ../../vdb1
lrwxrwxrwx 1 root root 10 Aug  1 11:12 dce5f167-fc11-45b0-8cf2-d16b8923a1b9 -> ../../vdb2
[root@node242 ~]# ln -s /dev/disk/by-partuuid/b0d06586-00e8-42fe-bdad-9b47abee374e /var/lib/ceph/osd/ceph-0/journal
[root@node242 ~]# ls -l /var/lib/ceph/osd/ceph-0/
total 0
lrwxrwxrwx 1 root root 58 Aug  1 11:19 journal -> /dev/disk/by-partuuid/b0d06586-00e8-42fe-bdad-9b47abee374e

創(chuàng)建 osd.0 的 keyring 文件

[root@node242 ~]# ceph-osd -i 0 --mkfs --mkkey
 HDIO_DRIVE_CMD(identify) failed: Inappropriate ioctl for device
2019-08-01 11:19:48.919616 7fd45e81d880 -1 journal check: ondisk fsid d1721d05-684d-483b-becb-5c07564f364c doesn't match expected 8558e48b-5363-4383-94c2-7f303676ddd5, invalid (someone else's?) journal
 HDIO_DRIVE_CMD(identify) failed: Inappropriate ioctl for device
 HDIO_DRIVE_CMD(identify) failed: Inappropriate ioctl for device
 HDIO_DRIVE_CMD(identify) failed: Inappropriate ioctl for device
2019-08-01 11:19:48.936239 7fd45e81d880 -1 filestore(/var/lib/ceph/osd/ceph-0) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory
2019-08-01 11:19:48.947600 7fd45e81d880 -1 created object store /var/lib/ceph/osd/ceph-0 journal /var/lib/ceph/osd/ceph-0/journal for osd.0 fsid 346436e3-a7f4-4fd9-9694-e9830faec4d5
2019-08-01 11:19:48.947663 7fd45e81d880 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory
2019-08-01 11:19:48.947757 7fd45e81d880 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring

查看

[root@node242 ~]# ls -l /var/lib/ceph/osd/ceph-0/
total 32
-rw-r--r-- 1 root root 37 Aug  1 11:19 ceph_fsid
drwxr-xr-x 4 root root 61 Aug  1 11:19 current
-rw-r--r-- 1 root root 37 Aug  1 11:19 fsid
lrwxrwxrwx 1 root root 58 Aug  1 11:19 journal -> /dev/disk/by-partuuid/b0d06586-00e8-42fe-bdad-9b47abee374e
-rw------- 1 root root 56 Aug  1 11:19 keyring
-rw-r--r-- 1 root root 21 Aug  1 11:19 magic
-rw-r--r-- 1 root root  6 Aug  1 11:19 ready
-rw-r--r-- 1 root root  4 Aug  1 11:19 store_version
-rw-r--r-- 1 root root 53 Aug  1 11:19 superblock
-rw-r--r-- 1 root root  2 Aug  1 11:19 whoami

添加 osd.0 的認證信息

[root@node242 ~]# ceph auth add osd.0 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-0/keyring
added key for osd.0

啟動服務(wù)

[root@node242 ~]# ceph-osd -i 0 
starting osd.0 at :/0 osd_data /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-0/journal
[root@node242 ~]# ps -ef | grep osd
root     19995     1  1 11:23 ?        00:00:00 ceph-osd -i 0
root     20090  9959  0 11:23 pts/0    00:00:00 grep --color=auto osd

把 osd.0 加入 crushmap

[root@node242 ~]# ceph osd tree
ID WEIGHT TYPE NAME    UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1      0 root default                                   
 0      0 osd.0             up  1.00000          1.00000
[root@node242 ~]# ceph osd crush add osd.0  1.0 root=default host=node242
add item id 0 name 'osd.0' weight 1 at location {host=node242,root=default} to crush map
[root@node242 ~]# ceph osd tree
ID WEIGHT  TYPE NAME        UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 1.00000 root default                                       
-2 1.00000     host node242                                   
 0 1.00000         osd.0         up  1.00000          1.00000

重復以上的創(chuàng)建 osd 過程叮雳，添加osd

[root@node242 ~]# ceph osd tree
ID WEIGHT  TYPE NAME        UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 3.00000 root default                                       
-2 3.00000     host node242                                   
 0 1.00000         osd.0         up  1.00000          1.00000 
 1 1.00000         osd.1         up  1.00000          1.00000 
 2 1.00000         osd.2         up  1.00000          1.00000

3、配置 Ceph 集群