前提
- 主機(jī)為CentOS 7.2及以上
- 預(yù)裝ceph版本為ceph-luminous 12.2.5
- 主節(jié)點(diǎn)可無密訪問其他兩個節(jié)點(diǎn)
- 搭建3臺主機(jī)形成的ceph cluster及cephfs
- 主節(jié)點(diǎn)配置本地ceph yum源
配置詳情如下：
| 節(jié)點(diǎn) | 安裝組件 |
|---|---|
| 主節(jié)點(diǎn) | ceph-deploy & mon & mgr & osd & ntp &mds &httpd & createrepo |
| 從節(jié)點(diǎn) | mon & mgr & osd & ntp &mds |
| 從節(jié)點(diǎn) | mon & mgr & osd & ntp &mds |
目錄結(jié)構(gòu)
[root@ceph-1 deploy]# ll
total 132
-rw-r--r-- 1 root root 97527 Jun 21 17:09 deploy.log
-rwxr-xr-x 1 root root 4288 Jun 21 17:02 deploy.sh
drwxr-xr-x 2 root root 4096 Jun 21 16:54 ntp
-rwxr-xr-x 1 root root 99 Jun 21 10:43 part.sh
drwxr-xr-x 2 root root 4096 Jun 20 15:57 py
drwxr-xr-x 2 root root 4096 Jun 20 10:46 rpm
drwxr-xr-x 2 root root 4096 Jun 21 16:39 test
drwxr-xr-x 2 root root 4096 Jun 21 16:54 yum
- rpm存放的是ceph相關(guān)的rpm包舟奠，用于建立本地源轴合，可參見搭建ceph本地源下載相關(guān)的rpm
- ntp存放主節(jié)點(diǎn)的ntp.conf和從節(jié)點(diǎn)的ntp1.conf
ntp.conf如下
driftfile /var/lib/ntp/drift
restrict default nomodify
restrict 127.0.0.1
restrict ::1
# Enable public key cryptography.
# #crypto
#
includefile /etc/ntp/crypto/pw
#
# # Key file containing the keys and key identifiers used when operating
# # with symmetric key cryptography.
keys /etc/ntp/keys
#
server 127.127.1.0
fudge 127.127.1.0 stratum 10
ntp1.conf
driftfile /var/lib/ntp/drift
restrict default nomodify
restrict 127.0.0.1
restrict ::1
# Enable public key cryptography.
#crypto
includefile /etc/ntp/crypto/pw
# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography.
keys /etc/ntp/keys
- yum中包含CentOS-Base.repo，epel.repo和ceph.repo
CentOS-Base.repo，epel.repo可參見Ceph 部署（Centos7 + Luminous）
ceph.repo
[ceph]
name=ceph
腳本 deploy.sh
#!/bin/bash
#配置開始
ceph_1_ip="192.168.16.4"#主節(jié)點(diǎn)ip
ceph_1_name="ceph-1"#主節(jié)點(diǎn)主機(jī)名
ceph_2_ip="192.168.16.5"#從節(jié)點(diǎn)ip
ceph_2_name="ceph-2"#從節(jié)點(diǎn)主機(jī)名
ceph_3_ip="192.168.16.6"#從節(jié)點(diǎn)ip
ceph_3_name="ceph-3"#從節(jié)點(diǎn)主機(jī)名
sub_network="192.168.16.0/24"#子網(wǎng)
ceph_1_dev=('vdb' 'vdc')#節(jié)點(diǎn)1 osd 目標(biāo)安裝磁盤
ceph_2_dev=('vdb' 'vdc')#節(jié)點(diǎn)2 osd 目標(biāo)安裝磁盤
ceph_3_dev=('vdb' 'vdc')#節(jié)點(diǎn)3 osd 目標(biāo)安裝磁盤
ceph_1_dev_journal=('vdd1' 'vdd2' 'vdd3' 'vdd4')#節(jié)點(diǎn)1 bluestore db&wal 目標(biāo)安裝磁盤
ceph_2_dev_journal=('vdd1' 'vdd2' 'vdd3' 'vdd4'))#節(jié)點(diǎn)2 bluestore db&wal 目標(biāo)安裝磁盤
ceph_3_dev_journal=('vdd1' 'vdd2' 'vdd3' 'vdd4'))#節(jié)點(diǎn)3 bluestore db&wal 目標(biāo)安裝磁盤
#配置結(jié)束
DEPLOY_DIR=$(cd `dirname $0`; pwd)
echo "$ceph_1_ip $ceph_1_name" >> /etc/hosts
echo "$ceph_2_ip $ceph_2_name" >> /etc/hosts
echo "$ceph_3_ip $ceph_3_name" >> /etc/hosts
ssh root@$ceph_2_name "echo '$ceph_1_ip $ceph_1_name' >> /etc/hosts;echo '$ceph_2_ip $ceph_2_name' >> /etc/hosts;echo '$ceph_3_ip $ceph_3_name' >> /etc/hosts"
ssh root@$ceph_3_name "echo '$ceph_1_ip $ceph_1_name' >> /etc/hosts;echo '$ceph_2_ip $ceph_2_name' >> /etc/hosts;echo '$ceph_3_ip $ceph_3_name' >> /etc/hosts"
#關(guān)閉防火墻
sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
setenforce 0
ssh root@$ceph_2_name "sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config;setenforce 0"
ssh root@$ceph_3_name "sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config;setenforce 0"
#搭建本地yum源頭
yum install httpd createrepo -y
mkdir -p /var/www/html/ceph/12.2.5
cp $DEPLOY_DIR/rpm/* /var/www/html/ceph/12.2.5
createrepo /var/www/html/ceph/12.2.5
#配置yum
yum clean all
echo "baseurl=http://$ceph_1_ip/ceph/12.2.5" >> $DEPLOY_DIR/yum/ceph.repo
echo "gpgcheck=0" >> $DEPLOY_DIR/yum/ceph.repo
\cp -fr $DEPLOY_DIR/yum/* /etc/yum.repos.d/
yum makecache
ssh root@$ceph_2_name "yum clean all"
ssh root@$ceph_3_name "yum clean all"
scp -r $DEPLOY_DIR/yum/* root@$ceph_2_name:/etc/yum.repos.d/
scp -r $DEPLOY_DIR/yum/* root@$ceph_3_name:/etc/yum.repos.d/
ssh root@$ceph_2_name "yum makecache"
ssh root@$ceph_3_name "yum makecache"
#安裝ntp
yum install ntp -y
\cp -fr $DEPLOY_DIR/ntp/ntp.conf /etc/
echo "server $ceph_1_ip" >> $DEPLOY_DIR/ntp/ntp1.conf
systemctl enable ntpd
systemctl restart ntpd
ssh root@$ceph_2_name "yum install ntp -y"
ssh root@$ceph_3_name "yum install ntp -y"
scp -r $DEPLOY_DIR/ntp/ntp1.conf root@$ceph_2_name:/etc/ntp.conf
scp -r $DEPLOY_DIR/ntp/ntp1.conf root@$ceph_3_name:/etc/ntp.conf
ssh root@$ceph_2_name "systemctl enable ntpd;systemctl restart ntpd"
ssh root@$ceph_3_name "systemctl enable ntpd;systemctl restart ntpd"
#安裝ceph
yum install ceph -y
ssh root@$ceph_2_name "yum install ceph -y"
ssh root@$ceph_3_name "yum install ceph -y"
#安裝ceph-deploy
yum install ceph-deploy -y
#部署ceph
mkdir ~/cluster
cd ~/cluster
CLUSTER_DIR=$(cd `dirname $0`; pwd)
ceph-deploy new $ceph_1_name $ceph_2_name $ceph_3_name
echo "public_network=$sub_network" >> ceph.conf
#echo "osd_crush_update_on_start = false" >> ceph.conf
ceph-deploy mon create-initial
ceph-deploy admin $ceph_1_name $ceph_2_name $ceph_3_name
#配置osd
index=0
for dev_name in ${ceph_1_dev[@]}
do
ceph-volume lvm zap /dev/$dev_name
ceph-deploy osd create $ceph_1_name --bluestore --data /dev/$dev_name --block-db /dev/${ceph_1_dev_journal[$index]} --block-wal /dev/${ceph_1_dev_journal[$index+1]}
index=$[$index+2]
done
index=0
for dev_name in ${ceph_2_dev[@]}
do
ssh root@$ceph_2_name "ceph-volume lvm zap /dev/$dev_name"
ceph-deploy osd create $ceph_2_name --bluestore --data /dev/$dev_name --block-db /dev/${ceph_2_dev_journal[$index]} --block-wal /dev/${ceph_2_dev_journal[$index+1]}
index=$[$index+2]
done
index=0
for dev_name in ${ceph_3_dev[@]}
do
ssh root@$ceph_3_name "ceph-volume lvm zap /dev/$dev_name"
ceph-deploy osd create $ceph_3_name --bluestore --data /dev/$dev_name --block-db /dev/${ceph_3_dev_journal[$index]} --block-wal /dev/${ceph_3_dev_journal[$index+1]}
index=$[$index+2]
done
#配置mgr
ceph-deploy mgr create $ceph_1_name $ceph_2_name $ceph_3_name
#啟動dashboard
ceph mgr module enable dashboard
#部署cephfs
ceph-deploy mds create $ceph_1_name $ceph_2_name $ceph_3_name
ceph osd pool create cephfs_data 128
ceph osd pool create cephfs_metadata 128
ceph fs new cephfs cephfs_metadata cephfs_data
mkdir /mnt/mycephfs
admin_key=`ceph auth get-key client.admin`
admin_key_base64=`ceph auth get-key client.admin |base64`
sleep 5#等待mds部署完成后再mount
mount -t ceph $ceph_1_name:6789,$ceph_2_name:6789,$ceph_3_name:6789:/ /mnt/mycephfs -o name=admin,secret=$admin_key
執(zhí)行腳本
nohup ./deploy.sh > deploy.log 2>&1 &
