工作隨筆 - shell腳本安裝鑒權(quán)版本zookeeper/kafka集群

記錄日常工作中的zk/kafka安裝

組件	版本	安裝包名
openjdk	1.8.0_345
zookeeper	3.7.1	apache-zookeeper-3.7.1-bin.tar.gz
kafka	2.12-2.5.1	kafka_2.12-2.5.1.tgz
centos	7.6

文件夾列表

.
├── apache-zookeeper-3.7.1-bin.tar.gz
├── install_zk_kafka.sh
├── jaas
│   ├── zk_client.conf
│   └── zk_server.conf
├── jaas.conf
├── java.env
├── kafka_2.12-2.5.1.tgz
└── kafka_client.properties

jaas.conf

KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="admin"
    password="pwd4test"
    user_admin="pwd4test"
    user_kafka="pwd4test";
};

ZkClient {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    username="kafka"
    password="pwd4test";

java.env

SERVER_JVMFLAGS="-Djava.security.auth.login.config=/home/test/App/zookeeper/conf/jaas/zk_server.conf -Dzookeeper.allowSaslFailedClients=false -Dzookeeper.sessionRequireClientSASLAuth=true"
CLIENT_JVMFLAGS="${CLIENT_JVMFLAGS} -Djava.security.auth.login.config=/home/test/App/zookeeper/conf/jaas/zk_client.conf"

kafka_client.properties

security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="pwd4test";

jaas/zk_client.conf

Client {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    username="kafka"
    password="pwd4test";
};

jaas/zk_server.conf

QuorumServer {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    user_admin="pwd4test";
};

QuorumLearner {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    username="admin"
    password="pwd4test";
};

Server{
    org.apache.zookeeper.server.auth.DigestLoginModule required
    user_kafka="pwd4test";
};

install_zk_kafka.sh

#!/bin/bash
# usage  : sh install_zk_kafka.sh zookeeper|kafka|clean

BASE_DIR=$(cd `dirname $0`; pwd)
ZK_PACKAGE="apache-zookeeper-3.7.1-bin.tar.gz"
ZK_SERVER=(192.168.1.1 92.168.1.2 92.168.1.3)
LOCAL_IP=$(ifconfig|grep inet|grep -v 100.100| sed -n '1p'|awk '{print $2}')
ZK_PORT="2181"
INSTALL_BASE_PATH="/home/test"
KAFKA_PACKAGE="kafka_2.12-2.5.1.tgz"
KAFKA_PORT="9092"
RUN_ID="test"

function log_error() {
    echo -e "\033[31m [ERROR] $@ \033[0m"
}

function log_info() {
    echo -e "\033[32m [INFO] $@ \033[0m"
}

function log_warn() {
    echo -e "\033[33m [WARN] $@ \033[0m"
}

function check_jdk() {
    java_version=$(java -version 2>&1|grep version|awk -F '"' '{print $2}')
    
    if [ "${java_version}"x == x ];then
        log_error "未找到JDK,請事先安裝JDK"
        exit 1
    else
        log_info "Java版本為:java_version"
    fi
    
}

function prepare_zk() {
    log_info "創(chuàng)建zookeeper所需文件夾"
    mkdir -pv ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs/zookeeper ${INSTALL_BASE_PATH}/Data/zookeeper    
}

function install_zk() {
    log_info "解壓安裝包${ZK_PACKAGE}"
    tar -xzvf ${BASE_DIR}/${ZK_PACKAGE} -C  ${INSTALL_BASE_PATH}/App/ 2>&1 > /dev/null
    mv ${INSTALL_BASE_PATH}/App/apache-zookeeper-3.7.1-bin ${INSTALL_BASE_PATH}/App/zookeeper

    log_info "創(chuàng)建配置文件${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg"
cat >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg <<EOF
tickTime=2000
initLimit=10
syncLimit=5
maxClientCnxns=500
autopurge.snapRetainCount=16
autopurge.purgeInterval=168
admin.enableServer=false
4lw.commands.whitelist=*
quorum.auth.enableSasl=true
quorum.auth.learnerRequireSasl=true
quorum.auth.serverRequireSasl=true
quorum.auth.learner.saslLoginContext=QuorumLearner
quorum.auth.server.saslLoginContext=QuorumServer
quorum.cnxn.threads.size=6
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
EOF

    echo "dataDir=${INSTALL_BASE_PATH}/Data/zookeeper" >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg
    echo "dataLogDir=${INSTALL_BASE_PATH}/Logs/zookeeper"  >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg
    echo "clientPort=${ZK_PORT}" >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg

    # get the server id
    server_id=""

    # update the server ip in config file
    i=0
    while [ $i -lt ${#ZK_SERVER[@]} ]
    do
        num=$i
        let i++
        echo "server.$i=${ZK_SERVER[$num]}:2888:3888" >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg
        [ "${LOCAL_IP}" == "${ZK_SERVER[$num]}" ] && log_info "server ID is $i" && server_id=$i && echo "$server_id" >> ${INSTALL_BASE_PATH}/Data/zookeeper/myid
    done

    # 判斷l(xiāng)ocal_ip是否屬于節(jié)點中的一個,否則中斷
    if [[ x"${server_id}" == "x"  ]];then
            log_error "腳本獲取的LOCAL_IP(${LOCAL_IP})不在列表${ZK_SERVER[@]}中,請手動設(shè)置LOCAL_IP變量"
        exit 1
    fi

    log_info "添加zookeeper到環(huán)境變量"
    echo "export ZK_HOME=${INSTALL_BASE_PATH}/App/zookeeper" >> /etc/profile
    echo "export PATH=\$ZK_HOME/bin:\$PATH" >> /etc/profile

    log_info "添加zookeeper鑒權(quán)文件"
    cp -r ${BASE_DIR}/jaas ${INSTALL_BASE_PATH}/App/zookeeper/conf/
    cp ${BASE_DIR}/java.env ${INSTALL_BASE_PATH}/App/zookeeper/conf/
    sed -ri "s#/home/finance#${INSTALL_BASE_PATH}#g" ${INSTALL_BASE_PATH}/App/zookeeper/conf/java.env
    chmod 755 ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs ${INSTALL_BASE_PATH}/Data
    chown -R ${RUN_ID}:${RUN_ID} ${INSTALL_BASE_PATH}/App/zookeeper ${INSTALL_BASE_PATH}/Logs/zookeeper ${INSTALL_BASE_PATH}/Data/zookeeper
}

function start_zk() {
    log_info "啟動zookeeper..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh start"
    sleep 5

    
    log_info "##########################"
    log_info "### 請記錄以下鑒權(quán)信息 ###"
    log_info "zookeeper鑒權(quán): kafka / pwd4test"
    log_info "##########################"
    log_warn "[注意]: 請在[第三個節(jié)點]按照完成之后,執(zhí)行命令查看啟動狀態(tài):source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh status"

}

function stop_zk() {
    log_info "停止zookeeper..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh stop"
    sleep 5
}

function status_zk() {
    log_info "zookeeper的status如下:"
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh status"
}

function prepare_kafka() {
    log_info "創(chuàng)建kafka所需文件夾"
    mkdir -pv ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs/kafka ${INSTALL_BASE_PATH}/Data/kafka    
}

function check_zk() {
    zk_status=$(source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh status)
    zk_mode=$(echo $zk_status | grep Mode)

    if [ "${zk_mode}"x == x ];then
        log_error "未找到zookeeper,請事先安裝zookeeper"
        exit 1
    else
        log_info "發(fā)現(xiàn)zookeeper運行,繼續(xù)執(zhí)行安裝kafka"
    fi

}    

function install_kafka() {
    log_info "解壓安裝包${KAFKA_PACKAGE}"
    tar -xzvf ${BASE_DIR}/${KAFKA_PACKAGE} -C  ${INSTALL_BASE_PATH}/App/ 2>&1 > /dev/null
    #mv ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1 ${INSTALL_BASE_PATH}/App/kafka

    log_info "創(chuàng)建配置文件${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties"

cat >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties <<EOF
delete.topic.enable=true
num.network.threads=5
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
num.partitions=3
default.replication.factor=3
min.insync.replicas=2
num.recovery.threads.per.data.dir=6
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=2
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=10000
auto.create.topics.enable=true
unclean.leader.election.enable=false
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN
allow.everyone.if.no.acl.found=true
EOF

    # get the server id
    server_id=""

    # update the server ip in config file
    i=0
    while [ $i -lt ${#ZK_SERVER[@]} ]
    do
        num=$i
        let i++
        [ "${LOCAL_IP}" == "${ZK_SERVER[$num]}" ] && echo "server ID is $i" && server_id=$i && echo "broker.id=${server_id}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    done

    # 判斷l(xiāng)ocal_ip是否屬于節(jié)點中的一個,否則中斷
    if [[ x"${server_id}" == "x"  ]];then
            log_error "腳本獲取的LOCAL_IP(${LOCAL_IP})不在列表${ZK_SERVER[@]}中,請手動設(shè)置LOCAL_IP變量"
        exit 1
    fi

    echo "listeners=SASL_PLAINTEXT://${LOCAL_IP}:${KAFKA_PORT}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    echo "advertised.listeners=SASL_PLAINTEXT://${LOCAL_IP}:${KAFKA_PORT}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    echo "log.dirs=${INSTALL_BASE_PATH}/Data/kafka" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    echo "zookeeper.connect=${ZK_SERVER[0]}:${ZK_PORT},${ZK_SERVER[1]}:${ZK_PORT},${ZK_SERVER[2]}:${ZK_PORT}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties

    log_info "創(chuàng)建kafka鑒權(quán)文件"
    cp  ${BASE_DIR}/jaas.conf  ${BASE_DIR}/kafka_client.properties ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/

    log_info "更改kafka日志路徑為: ${INSTALL_BASE_PATH}/Logs/kafka"
    sed -ri "s#LOG_DIR=\".*#LOG_DIR=\"${INSTALL_BASE_PATH}\/Logs\/kafka\"#g" ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-run-class.sh

    log_info "修改配置,開啟kafka鑒權(quán)"
    sed -ri "s#KAFKA_OPTS=\"\"#KAFKA_OPTS=\"$KAFKA_OPTS -Djava.security.auth.login.config=${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/jaas.conf\"#g" ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-run-class.sh
    sed -ri "s#KAFKA_JVM_PERFORMANCE_OPTS=\"-server#KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15 -Djava.awt.headless=true -Dzookeeper.sasl.client=true -Dzookeeper.sasl.clientconfig=ZkClient -Dzookeeper.sasl.client.username=kafka -Djava.security.auth.login.config=${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/jaas.conf#g" ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-run-class.sh


    chmod 755 ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs ${INSTALL_BASE_PATH}/Data
    chown -R ${RUN_ID}:${RUN_ID} ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1 ${INSTALL_BASE_PATH}/Logs/kafka ${INSTALL_BASE_PATH}/Data/kafka
}

function start_kafka() {
    log_info "啟動kafka..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-server-start.sh -daemon ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties"
    sleep 5

    log_info "##########################"
    log_info "### 請記錄以下鑒權(quán)信息 ###"
    log_info "kafka鑒權(quán): admin / pwd4test"
    log_info "##########################"

    log_info "待三個節(jié)點啟動完成,自行創(chuàng)建topic..."
}

function stop_kafka() {
    log_info "停止kafka..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-server-stop.sh"
    sleep 5
}

function clean() {
    log_warn "清理zookeeper..."
    rm -rf ${INSTALL_BASE_PATH}/App/zookeeper ${INSTALL_BASE_PATH}/Logs/zookeeper ${INSTALL_BASE_PATH}/Data/zookeeper
    log_warn "清理kafka..."
    rm -rf ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1 ${INSTALL_BASE_PATH}/Logs/kafka ${INSTALL_BASE_PATH}/Data/kafka
    log_warn "清理完成..."
}

case "$1" in
  zookeeper)
        check_jdk
        prepare_zk
        install_zk
        start_zk
        #status_zk
        ;;
  kafka)
        check_jdk
        check_zk
        prepare_kafka
        install_kafka
        start_kafka
        ;;
  clean)
        stop_kafka
        stop_zk
        clean
        ;;        
  *)
        echo $"Usage: $0 {zookeeper|kafka|clean}"
        exit 2
esac

exit 0

使用方法

# 修改腳本中IP列表倔矾，確保三臺服務(wù)器IP列表順序一致
ZK_SERVER=(192.168.1.1 92.168.1.2 92.168.1.3)
# 三臺服務(wù)器依次安裝zookeeper
sh install_zk_kafka.sh zookeeper
# 待三臺zookeeper安裝完成爱态，再依次安裝kafka
sh install_zk_kafka.sh kafka

# [慎重執(zhí)行]清理操作席覆，清理所有zookeeper/kafka安裝目錄,以便重裝
sh install_zk_kafka.sh clean

# zookeeper鑒權(quán): kafka / pwd4test
# kafka鑒權(quán): admin / pwd4test以下為純手動安裝配置方式

最后編輯于：2022.11.18 15:32:03

?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請聯(lián)系作者

人面猴
序言：七十年代末，一起剝皮案震驚了整個濱河市吊履，隨后出現(xiàn)的幾起案子，更是在濱河造成了極大的恐慌，老刑警劉巖获高，帶你破解...
沈念sama閱讀 221,548評論 6贊 515
死咒
序言：濱河連續(xù)發(fā)生了三起死亡事件，死亡現(xiàn)場離奇詭異吻育，居然都是意外死亡念秧，警方通過查閱死者的電腦和手機，發(fā)現(xiàn)死者居然都...
沈念sama閱讀 94,497評論 3贊 399
救了他兩次的神仙讓他今天三更去死
文/潘曉璐我一進店門布疼，熙熙樓的掌柜王于貴愁眉苦臉地迎上來摊趾，“玉大人，你說我怎么就攤上這事游两±悖” “怎么了？”我有些...
開封第一講書人閱讀 167,990評論 0贊 360
道士緝兇錄：失蹤的賣姜人
文/不壞的土叔我叫張陵贱案，是天一觀的道長肛炮。經(jīng)常有香客問我，道長宝踪，這世上最難降的妖魔是什么侨糟？我笑而不...
開封第一講書人閱讀 59,618評論 1贊 296
?港島之戀（遺憾婚禮）
正文為了忘掉前任，我火速辦了婚禮瘩燥，結(jié)果婚禮上秕重，老公的妹妹穿的比我還像新娘。我一直安慰自己厉膀，他們只是感情好溶耘，可當我...
茶點故事閱讀 68,618評論 6贊 397
惡毒庶女頂嫁案：這布局不是一般人想出來的
文/花漫我一把揭開白布二拐。她就那樣靜靜地躺著，像睡著了一般凳兵。火紅的嫁衣襯著肌膚如雪百新。梳的紋絲不亂的頭發(fā)上，一...
開封第一講書人閱讀 52,246評論 1贊 308
城市分裂傳說
那天庐扫，我揣著相機與錄音吟孙，去河邊找鬼。笑死聚蝶，一個胖子當著我的面吹牛杰妓，可吹牛的內(nèi)容都是我干的。我是一名探鬼主播碘勉，決...
沈念sama閱讀 40,819評論 3贊 421
雙鴛鴦連環(huán)套：你想象不到人心有多黑
文/蒼蘭香墨我猛地睜開眼巷挥，長吁一口氣：“原來是場噩夢啊……” “哼！你這毒婦竟也來了验靡？” 一聲冷哼從身側(cè)響起倍宾，我...
開封第一講書人閱讀 39,725評論 0贊 276
萬榮殺人案實錄
序言：老撾萬榮一對情侶失蹤，失蹤者是張志新（化名）和其女友劉穎胜嗓，沒想到半個月后高职，有當?shù)厝嗽跇淞掷锇l(fā)現(xiàn)了一具尸體，經(jīng)...
沈念sama閱讀 46,268評論 1贊 320
?護林員之死
正文獨居荒郊野嶺守林人離奇死亡辞州，尸身上長有42處帶血的膿包…… 初始之章·張勛以下內(nèi)容為張勛視角年9月15日...
茶點故事閱讀 38,356評論 3贊 340
?白月光啟示錄
正文我和宋清朗相戀三年怔锌，在試婚紗的時候發(fā)現(xiàn)自己被綠了。大學時的朋友給我發(fā)了我未婚夫和他白月光在一起吃飯的照片变过。...
茶點故事閱讀 40,488評論 1贊 352
活死人
序言：一個原本活蹦亂跳的男人離奇死亡埃元，死狀恐怖，靈堂內(nèi)的尸體忽然破棺而出媚狰，到底是詐尸還是另有隱情岛杀，我是刑警寧澤，帶...
沈念sama閱讀 36,181評論 5贊 350
?日本核電站爆炸內(nèi)幕
正文年R本政府宣布崭孤，位于F島的核電站类嗤，受9級特大地震影響，放射性物質(zhì)發(fā)生泄漏辨宠。R本人自食惡果不足惜遗锣，卻給世界環(huán)境...
茶點故事閱讀 41,862評論 3贊 333
男人毒藥：我在死后第九天來索命
文/蒙蒙一、第九天我趴在偏房一處隱蔽的房頂上張望彭羹。院中可真熱鬧黄伊，春花似錦泪酱、人聲如沸派殷。這莊子的主人今日做“春日...
開封第一講書人閱讀 32,331評論 0贊 24
一樁弒父案，背后竟有這般陰謀
文/蒼蘭香墨我抬頭看了看天上的太陽毡惜。三九已至拓轻，卻和暖如春，著一層夾襖步出監(jiān)牢的瞬間经伙，已是汗流浹背扶叉。一陣腳步聲響...
開封第一講書人閱讀 33,445評論 1贊 272
情欲美人皮
我被黑心中介騙來泰國打工，沒想到剛下飛機就差點兒被人妖公主榨干…… 1. 我叫王不留帕膜，地道東北人枣氧。一個月前我還...
沈念sama閱讀 48,897評論 3贊 376
代替公主和親
正文我出身青樓，卻偏偏與公主長得像垮刹，于是被迫代替她去往敵國和親达吞。傳聞我的和親對象是個殘疾皇子，可洞房花燭夜當晚...
茶點故事閱讀 45,500評論 2贊 359

工作隨筆 - shell腳本安裝鑒權(quán)版本zookeeper/kafka集群

文件夾列表

jaas.conf

java.env

kafka_client.properties

jaas/zk_client.conf

jaas/zk_server.conf

install_zk_kafka.sh

使用方法

推薦閱讀更多精彩內(nèi)容