腳本位置：

zice服務(wù)器

/home/jerry/aliexpension/trigger.py  #服務(wù)器創(chuàng)建觸發(fā)器腳本
/home/jerry/aliexpension/deploy_mobile.sh  #自動部署腳本
/home/jerry/aliexpension/phj-web-aliexpansion.jar  #鄭敏的創(chuàng)建服務(wù)器腳本

腳本通過調(diào)用zabbix的API獲取監(jiān)控服務(wù)器的CPU段只、內(nèi)存和磁盤的觸發(fā)器狀態(tài)

狀態(tài)異常則觸發(fā)trigger.py腳本剥懒，調(diào)用鄭敏的金融云服務(wù)器創(chuàng)建腳本進行服務(wù)器的創(chuàng)建并調(diào)用部署部署shell腳本部署生產(chǎn)代碼瞧栗。

處理流程

服務(wù)器CPU宗挥、內(nèi)存索昂、磁盤其中任何一項發(fā)生警報-->觸發(fā)trigger.py腳本-->調(diào)用鄭敏服務(wù)器創(chuàng)建腳本窝撵，創(chuàng)建完畢服務(wù)器-->調(diào)用自動部署腳本部署完畢蝶缀。

trigger.py腳本報警邏輯

• 當服務(wù)器異常導(dǎo)致觸發(fā)創(chuàng)建服務(wù)器的動作后丹喻，等待30s進行服務(wù)器創(chuàng)建
• 在創(chuàng)建完畢第一個服務(wù)器后的6個小時內(nèi)不會再進行創(chuàng)建服務(wù)器
• 若服務(wù)器異常一直持續(xù)了6個小時，則會在6個小時后創(chuàng)建第二個服務(wù)器翁都，邏輯返回到第一步碍论，以此類推。

trigger.py 腳本內(nèi)容：

vim /home/jerry/aliexpension/trigger.py

#導(dǎo)入腳本運行所需模塊
import requests, json, time, pickle, os
#獲取當前時間
def getcurrenttime(arg):
    if arg == 's':
        t = time.time()
    elif arg == 'd':
        t = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    else:
        print(arg+'時間獲取出錯')
    return t
headers = {'Content-Type': 'application/json-rpc'}
url = 'http://zabbix.puhuijia.com/zabbix/api_jsonrpc.php'
#url = 'http://10.2.2.19/pxzabbix/api_jsonrpc.php' '''測試環(huán)境地址'''
'''獲取驗證zabbix身份信息的token柄慰，返回token值'''
def getToken(username='zhaoliang', passwd='zhaoliang123'):
    '''獲取登陸zabbixapi的token信息鳍悠，返回token的值'''
    data = {
        "jsonrpc": "2.0",
        "method": "user.login",
        "params": {
            "user": username,
            "password": passwd
        },
        "id": 1
    }
    request = requests.post(url=url, headers=headers,data=json.dumps(data))
    dict = json.loads(request.text)
    return dict['result']

'''獲取某個服務(wù)器的某個觸發(fā)器的狀態(tài)信息税娜，返回0或者1，0表示狀態(tài)正常藏研，1表示狀態(tài)異常'''
def getstatus(token,hostid,triggerid):
    '''獲取服務(wù)器監(jiān)控項item的狀態(tài),即是否被觸發(fā),被觸發(fā)返回1,沒被觸發(fā)返回0'''
    data = {
        "jsonrpc": "2.0",
        "method": "trigger.get",
        "params": {
            "triggerids": triggerid,
            "hostids": hostid,
            "output": "extend",
            "selectFunctions": "extend"
        },
        "id": 1,
        "auth": token,
    }
    request = requests.post(url=url, headers=headers, data=json.dumps(data))
    dict = json.loads(request.text)
    return dict['result'][0]['value']

'''要監(jiān)控的zabbix服務(wù)器信息'''
def monitoredhost(token=getToken()):
    '''通過myhost和triggername兩個列表來生成一個字典敬矩，它包含了我們所有需要監(jiān)控的服務(wù)器的監(jiān)控項,返回所有服務(wù)器信息字典'''
    myhost = ['dubbo1','dubbo2','dubbo3','dubbo4','dubbo5','dubbo6','dubbo7','dubbo8','dubbo9','dubbo10','web-mobile1','web-mobile2','web_back1','web_back2']
    triggername = ["{HOST.NAME}服務(wù)器可用內(nèi)存不足","{HOST.NAME}處理器負載過高","本機tomcat已3分鐘不可用"]
    data = {
        "jsonrpc": "2.0",
        "method": "host.get",
        "params": {
            "output": ["host","hostid"],
        },
        "id": 1,
        "auth": token
    }
    request = requests.post(url=url, headers=headers, data=json.dumps(data))
    hosts = json.loads(request.text)
    items = {}
    for i in hosts['result']:
        hostid = i['hostid']
        hostname = i['host']
        if hostname in myhost:
            items[hostname] = {"id":hostid, "triggers":{}}
            data = {
                "jsonrpc": "2.0",
                "method": "trigger.get",
                "params": {
                    'hostids': hostid
                },
                "id": 1,
                "auth": token
            }
            request = requests.post(url=url, headers=headers, data=json.dumps(data))
            dict = json.loads(request.text)
            for m in dict['result']:
                if m["description"] == triggername[0]:
                    items[hostname]["triggers"]["mem"] = m["triggerid"]
                elif m["description"] == triggername[1]:
                    items[hostname]["triggers"]["cpu"] = m["triggerid"]
                elif m["description"] == triggername[2]:
                    items[hostname]["triggers"]["heap"] = m["triggerid"]
                else:
                    continue

    return  items
def findall():
    '''返回異常服務(wù)器信息的字典,若無異常則返回0'''
    excep = {}
    error = 0
    items = monitoredhost()
    for host in items:
        excep[host] = {}
        for trigger in items[host]['triggers']:
            hostid = items[host]['id']
            triggerid = items[host]['triggers'][trigger]
            status = getstatus(token=getToken(),hostid=hostid,triggerid=triggerid)
            if status == str(0):
                '''狀態(tài)正常error=0'''
                print('{time} INFO:主機---> {0} 的觸發(fā)器 {1} 狀態(tài)為 {2} 正常'.format(host, trigger, status,time=getcurrenttime('d')))
            elif status == str(1):
                '''狀態(tài)異常error=1,隨后返回異常服務(wù)器信息的字典'''
                error = 1
                print('{time} INFO:主機---> {0} 的觸發(fā)器 {1} 狀態(tài)為 {2} 發(fā)生異常'.format(host, trigger, status,time=getcurrenttime('d')))
                excep[host][trigger] = triggerid
            else:
                return 1
    if error == 1:
        return excep
    else:
        return 0

def createcheck(hostname,hoststatus=1):
    '''檢查服務(wù)器當前狀態(tài)是否需要創(chuàng)建新的服務(wù)器,返回0表示需要進行服務(wù)器創(chuàng)建,返回1表示無需創(chuàng)建服務(wù)器'''
    status = 1
    if os.path.isfile('hoststatus.pkl'):
        '''文件已存在'''
        pass
    else:
        '''新建pkl文件'''
        f = open('hoststatus.pkl', 'w')
        f.close()
    if os.path.getsize('hoststatus.pkl') != 0:
        with open('hoststatus.pkl', 'rb') as fr:
            s = pickle.load(fr)
            s1 = s.copy()
        with open('hoststatus.pkl', 'wb') as f:
            if hostname in s1:
                s1[hostname]['time2'] = getcurrenttime('s')
                timecha = s1[hostname]['time2'] - s1[hostname]['time1']
                if timecha >= 30 and s1[hostname]['hoststatus'] == 1:
                    s1[hostname]['hoststatus'] = 0
                    print('{time} 首次觸發(fā)創(chuàng)建服務(wù)器 {0}'.format(hostname, time=getcurrenttime('d')))
                    status =  0
                elif timecha >=36000 and s1[hostname]['hoststatus'] == 0:
                    s1[hostname]['hoststatus'] = 1
                    s1[hostname]['time1'] = getcurrenttime('s')
                    print('{time} 異常已持續(xù)6小時,再次觸發(fā)創(chuàng)建服務(wù)器 {0}'.format(hostname, time=getcurrenttime('d')))
                    status = 0
                elif timecha < 30 and s1[hostname]['hoststatus'] == 1:
                    print('{time} 主機 {0} 正在等待30s'.format(hostname, time=getcurrenttime('d')))
                    status = 1
                elif timecha < 30 and s1[hostname]['hoststatus'] == 0:
                    print('{time} 主機 {0} 出現(xiàn)異常ERROR'.format(hostname, time=getcurrenttime('d')))
                    status = 1
                elif 36000 > timecha >= 30 and s1[hostname]['hoststatus'] == 0:
                    print('{time} 主機 {0} 狀態(tài)異常且在監(jiān)控中, 持續(xù)時間為 {1} 分鐘, {2} 分鐘后若持續(xù)保持異常狀態(tài)則會新建服務(wù)器.'.format(hostname,int(timecha/60),int(600-timecha/60),time=getcurrenttime('d')))
                    status = 1

            else:
                print('{time} 主機 {0} 首次出現(xiàn)異常,已添加到文件中.'.format(hostname, time=getcurrenttime('d')))
                s1[hostname] = {}
                s1[hostname]['time1'] = getcurrenttime('s')
                s1[hostname]['hoststatus'] = 1
                print('{time} 服務(wù)器狀態(tài)異常等待，若持續(xù)30s則創(chuàng)建服務(wù)器遥倦，等待30s......'.format(time=getcurrenttime('d')))
                status = 1
            pickle.dump(s1, f)
            return status

    else:
        print('pkl文件大邪：{0} 為空,初始化pkl文件'.format(os.path.getsize('hoststatus.pkl')))
        time = getcurrenttime('s')
        info = {}
        info[hostname] = {}
        info[hostname]['time1'] = time
        info[hostname]['hoststatus'] = 0
        with open('hoststatus.pkl', 'wb') as f1:
            pickle.dump(info, f1)
        print('{time} 服務(wù)器狀態(tài)異常等待，若持續(xù)30s則創(chuàng)建服務(wù)器袒哥，等待30s......'.format(time=getcurrenttime('d')))
    return 1

def dubboapi():
    '''調(diào)用阿里云自動擴容dubbo服務(wù)器的url'''
    requests.get('http://localhost:7777/create_ecs_instance?instance_type=1')
    print('{time} 創(chuàng)建新的dubbo服務(wù)器完畢缩筛！'.format(time=getcurrenttime('d')))
def webmobileapi():
    '''調(diào)用阿里云自動擴容web服務(wù)器的url'''
    requests.get('http://localhost:7777/create_ecs_instance?instance_type=2')
    print('{time} 創(chuàng)建新的web服務(wù)器完畢！'.format(time=getcurrenttime('d')))


def createserver():
    '''主體函數(shù),用來將所有函數(shù)串聯(lián)起來'''
    print('{time} 開始進行狀態(tài)檢查.'.format(time=getcurrenttime('d')))
    result = findall()
    if result == 0:
        print('{time} 本次狀態(tài)檢查沒有發(fā)現(xiàn)有異常.'.format(time=getcurrenttime('d')))
    elif result == 1:
        print('{time} 本次狀態(tài)檢查沒有發(fā)現(xiàn)有異常,返回值為1.'.format(time=getcurrenttime('d')))
    else:
        for i in result:
            if 'dubbo' in i and result[i].get('trigger', 0):
                if createcheck(i, hoststatus=1) == 0:
                    for j in result[i]:
                        print('{time} {i} 服務(wù)器 {j} 參數(shù)值超過閾值,準備新建服務(wù)器.'.format(time=getcurrenttime('d'), j=j, i=i))
                    dubboapi()
                else:
                    continue
            elif 'mobile' in i and result[i].get('trigger', 0):
                if createcheck(i, hoststatus=1) == 0:
                    for j in result[i]:
                        print('{time} {i} 服務(wù)器 {j} 參數(shù)值超過閾值,準備新建服務(wù)器.'.format(time=getcurrenttime('d'), j=j, i=i))
                    webmobileapi()
                else:
                    continue
            elif 'web_back' in i and result[i].get('trigger', 0):
                pass

while True:
    createserver()
    time.sleep(30)

deploy_mobile.sh 腳本內(nèi)容

vim /home/jerry/aliexpension/deploy_mobile.sh

#!/bin/bash
# script usage:
#command line------> sh scriptname.sh IP TYPE
#DESCRIPTION
#           scriptname.sh:  the name of this script
#           IP:  the IP address of new server
#           TYPE:
#                1    the project type of web
#                2    the project type of dubbo

dubbo() {
hostname=dubbo`date +%m%d%H%M%S`
ssh -o StrictHostKeyChecking=no -p 20022 $1 "ls"
scp -P 20022 /etc/init.d/dubbo $1:/home/jerry/
ssh -o StrictHostKeyChecking=no -p 20022 $1 "/bin/expect <<EOF
spawn su - root
expect \"assword\"
send \"Puhuijia@123\r\"
send \"hostnamectl set-hostname ${hostname}\r\"
send \"mkdir -p /alidata/puhuijia/puhuijia_mobile_log/ /dubbo_provider/log /alidata/puhuijia/puhuijia_mobile_properties /dubbo_provider/service_mobile\r\"
send \"chown -R jerry:jerry /dubbo_provider/\r\"
send \"chown -R jerry:jerry /alidata/\r\"
send \"rm -rf /etc/init.d/dubbo\r\"
send \"cp -rf /home/jerry/dubbo /etc/init.d/\r\"
send \"echo $1 $hostname >> /etc/hosts\r\"
expect eof
EOF"
ssh -p 20022 $1 "rm -rf /dubbo_provider/service_mobile/*"
rsync -e  'ssh -p 20022' -avz /dubbo_provider/service_mobile/ $1:/dubbo_provider/service_mobile/
rsync -e  'ssh -p 20022' -avz /home/jerry/aliexpension/puhuijia_mobile_properties/ $1:/alidata/puhuijia/puhuijia_mobile_properties/
ssh -p 20022 $1 "/etc/init.d/dubbo service_mobile restart"
}
web() {
hostname=web_mobile`date +%m%d%H%M%S`
ssh -o StrictHostKeyChecking=no -p 20022 $1 "ls"
scp -P 20022 /etc/init.d/tomcat $1:/home/jerry/
ssh -o StrictHostKeyChecking=no -p 20022 $1 "/bin/expect <<EOF
spawn su - root
expect \"assword\"
send \"Puhuijia@123\r\"
send \"hostnamectl set-hostname ${hostname}\r\"
send \"mkdir -p /alidata/puhuijia/puhuijia_mobile_log/ /alidata/puhuijia/puhuijia_mobile_properties\r\"
send \"tar -zxvf /usr/local/soft/apache-tomcat-8.0.36.tar.gz -C /usr/local/\r\"
send \"mv -f /usr/local/apache-tomcat-8.0.36 /usr/local/tomcat_mobile\r\"
send \"chown -R jerry:jerry /usr/local/tomcat_mobile\r\"
send \"rm -rf /etc/init.d/tomcat\r\"
send \"cp -rf /home/jerry/tomcat /etc/init.d/\r\"
send \"echo $1 $hostname >> /etc/hosts\r\"
expect eof
EOF"
ssh -p 20022 $1 "rm -rf /usr/local/tomcat_mobile/webapps/*"
rsync -e 'ssh -p 20022' -avz /usr/local/tomcat_zice/webapps/app $1:/usr/local/tomcat_mobile/webapps/
rsync -e  'ssh -p 20022' -avz /home/jerry/aliexpension/puhuijia_mobile_properties/ $1:/alidata/puhuijia/puhuijia_mobile_properties/
ssh -p 20022 $1 "cd /usr/local/tomcat_mobile/bin;mv catalina.sh catalina.sh.bak${hostname};wget https://coding.net/u/himan/p/staticfile/git/raw/master/catalina.sh;sed -i  's/\r//' catalina.sh;chmod 755 catalina.sh"
ssh -p 20022 $1 "/etc/init.d/tomcat tomcat_mobile restart"
ssh -p 20022 $1 "/etc/init.d/tomcat tomcat_mobile restart"
}



case $2 in
1)
  web $1
;;
2)
  dubbo $1
;;
esac