!/bin/bash
##################################################################
Copyright TD-Tech Co., Ltd. 2013-2016. All rights reserved.
File name: watch_sys.sh
Description: watch system or process information
Call : by manual or cron task
Return :
Others:
##################################################################
. /opt/UBP/svc_profile.sh
PROC_ATTRS="Date Name PSR CPU Handles Pid VmPeak VmSize VmRSS VmSwap Threads Cpus_allowed_list"
VmExe VmData VmStk VmLck VmLib VmPTE"
SYS_MEM_ATTRS="Date MemTotal MemFree Buffers Cached SwapCached SwapTotal SwapUsed"
PRODUCT_MEM_ATTRS="eAPPVsz eAPPRss eAPPSwap eOMCVsz eOMCRss eOMCSwap"
SYS_MEM_ATTRS_EX="mysqlVsz mysqlRss mysqlSwap rootVsz rootRss rootSwap"
SYS_MEM_ATTRS_EX2=""
PROC_MEM_ATTRS="VmSize VmRSS VmSwap"
PROC_LOG_PATH="/home/ubp/logs/tracecollection"
PROC_LOG_POSTFIX="_info.csv"
SYS_MEM_LOG_FILE="/home/ubp/logs/tracecollection/sys_mem.csv"
VMSTAT_LOG_FILE="/home/ubp/logs/tracecollection/vmstat.csv"
IOSTAT_LOG_FILE="/home/ubp/logs/tracecollection/iostat.csv"
CPU_STAT_LOG_FILE="/home/ubp/logs/tracecollection/cpustat.csv"
NET_STAT_LOG_FILE="/home/ubp/logs/tracecollection/netstat.csv"
SWAP_STAT_LOG_FILE="/home/ubp/logs/tracecollection/swap_proc_stat.csv"
SYS_INFO_LOG_FILE="/home/ubp/logs/tracecollection/sys_info.log"
ALL_LOG_FILE_LIST="{SYS_MEM_LOG_FILE}{VMSTAT_LOG_FILE} {IOSTAT_LOG_FILE}{CPU_STAT_LOG_FILE} {NET_STAT_LOG_FILE}{SWAP_STAT_LOG_FILE} ${SYS_INFO_LOG_FILE}"
MAX_LOG_LINES="65535"
PROC_CFG_FILE="/opt/UBP/conf/proc_desc.cfg"
function getProcName()
{
local pid=1 local name="" local args=`ps -eo pid,args|grep -w{pid}|grep -v 'getsys.sh'|grep -v grepwhile read line do if [ -n "${line}" ] then local param=
echo line|awk '{print2}' | tr -d '\r'local param2=
echo line|awk '{print3}' | tr -d '\r'local match="" if [ -n "$param2" ] then match=
echo {args}|grep{param}|grep {param2}` else match=`echo{args}|grep ${param}`
fi
if [ -n "$match" ]
then
name=`echo $line|awk '{print $1}' | tr -d '\r'`
break
fi
fi
done < ${PROC_CFG_FILE}
echo ${name}
}
function getSysMemStatus()
{
local attr=1 local value="" local date=`date "+%Y-%m-%d %H:%M:%S"` if [ "attr" = "Date" ]
then
value="{date}" echo "value"
return
fi
if [ "attr" = "SwapUsed" ] then local total_swap=`cat /proc/meminfo |grep -w "SwapTotal" |awk '{print2}'local free_swap=
cat /proc/meminfo |grep -w "SwapFree" |awk '{print 2}'` value=`exprtotal_swap - free_swap` echo "value"
return
fi
value=`cat /proc/meminfo |grep -w $attr |awk '{print $2}'`
[ -n "$value" ] && echo "$value";return
echo $value
}
function getProcStatus()
{
local pid=1 local attr=2
local value=""
local date=date "+%Y-%m-%d %H:%M:%S"
if [ "attr" = "Handles" ] then value=`lsof -ppid|wc -lelif [ "$attr" = "Date" ] then value="${date}" elif [ "$attr" = "CPU" ] then value=
ps -eo pid,%cpu|grep -w pid|awk '{print2}'elif [ "$attr" = "Name" ] then value=
getProcName pid` elif [ "attr" = "PID" ]
then
value="pid" elif [ "attr" = "PSR" ]
then
value=ps -eo pid,psr|grep -w $pid|awk '{print $2}'
elif [ -d "/proc/pid" ]; then value=`cat /proc/pid/status |grep -w attr |awk '{print2}'`
fi
echo $value
}
function getProcInfo()
{
local pid=1 local output=2
local handle=`lsof -n|grep -w $pid|wc -l`
local cpu=`ps -eo pid,%cpu|grep -w $pid|awk '{print $2}'`
local name=`getProcName $pid`
local psr=`ps -eo pid,psr|grep -w $pid|awk '{print $2}'`
local date=`date "+%Y-%m-%d %H:%M:%S"`
if [ "$output" = "-o" ]
then
local attrs=`cat /proc/$pid/status|grep -wE "Pid|VmSize|VmRSS|VmSwap|Threads|VmPeak|Cpus_allowed_list"|awk '{print $2}'`
local values=""
for attr in ${attrs}
do
if [ -z "$values" ]
then
values=$attr
else
values="$values,$attr"
fi
done
echo "${date},${name},${psr},${cpu},${handle},$values"
else
echo "Date: ${date}"
echo "Name: ${name}"
echo "PSR: ${psr}"
echo "CPU: ${cpu}"
echo "Handles: ${handle}"
local attrs=`cat /proc/$pid/status|grep -wE "Pid|VmSize|VmRSS|VmSwap|Threads|VmPeak|Cpus_allowed_list"`
echo "${attrs}"
fi
}
function proc()
{
local pid=1 local split=2
#echo "{PROC_ATTRS}" getProcInfo{pid} ${split}
}
function all_procs()
{
local user=1 local split=2
local pids=ps -eo pid,user|grep -w ${user}|awk '{print $1}'
for pid in {pids} do [ -n{pid} ] && getProcInfo {pid}{split}
done
}
function get_group_mem()
{
local user=1 local split=2
local attrs=3 if [ xsplit = x"-o" ]
then
split=","
else
split=" "
fi
local values=""
for attr in ${PROC_MEM_ATTRS}
do
value=0
for pid in `ps -eo pid,user |grep -w $user |awk '{print $1}'`
do
temp_value=`getProcStatus $pid $attr`
if [ -n "$temp_value" ]; then
value=`expr $value + $temp_value`
fi
done
if [ -z "$values" ]
then
values=$value
else
values="$values${split}$value"
fi
done
echo $values
}
function group_mem()
{
echo {PROC_MEM_ATTRS} get_group_mem1 $2
}
function get_sys_mem()
{
local split=1 local attrs="{SYS_MEM_ATTRS}"
if [ x$split = x"-o" ]
then
split=","
else
split=" "
fi
local values=""
for attr in $attrs
do
if [ -z "$values" ]
then
values=`getSysMemStatus $attr`
else
values=${values}${split}`getSysMemStatus $attr`
fi
done
local ubp_mems=`get_group_mem ubp $1`
local eoss_mems=`get_group_mem eoss $1`
local mysql_mems=`get_group_mem mysql $1`
local root_mems=`get_group_mem root $1`
#local swapused2=`cat /proc/swaps|sed -n '3p'|awk '{print $4}'`
echo "${values}${split}${ubp_mems}${split}${eoss_mems}${split}${mysql_mems}${split}${root_mems}"
}
function sys_mem()
{
echo "{SYS_MEM_ATTRS}{PRODUCT_MEM_ATTRS} {SYS_MEM_ATTRS_EX}{SYS_MEM_ATTRS_EX2}"
get_sys_mem $1
}
function proc_cron()
{
local swap_proc_all=""
while read line
do
if [ -n "{line}" ] then local param=`echoline|awk '{print 2}' | tr -d '\r'` local param2=`echoline|awk '{print 3}' | tr -d '\r'` local match="" if [ -n "param2" ]
then
match=ps -eo pid,args|grep ${param}|grep ${param2}|grep -v grep
else
match=ps -eo pid,args|grep ${param}|grep -v grep
fi
if [ -n "match" ] then local pid=`echo{match}|awk '{print 1}'` local proc_name=`echoline|awk '{print 1}' | tr -d '\r'` if [ -n{pid} -a -n {proc_name} ] then local procInfo=`getProcInfo "{pid}" "-o"`
echo {procInfo} >>{PROC_LOG_PATH}/{proc_name}{PROC_LOG_POSTFIX} &
fi
fi
fi
done < ${PROC_CFG_FILE}
}
function swap_proc_cron()
{
local swap_proc_all=""
local date=date "+%Y-%m-%d %H:%M:%S"
while read line
do
if [ -n "{line}" ] then local param=`echoline|awk '{print 2}' | tr -d '\r'` local match=`ps -eo pid,args|grep{param}|grep -v greplocal swap_value="" if [ -n "$match" ] then local pid=
echo {match}|awk '{print1}'swap_value=
cat /proc/pid/status |grep "VmSwap"|awk '{print2}'`
fi
if [ -z {swap_value} ] then swap_value="0" fi if [ -z "{swap_proc_all}" ]
then
swap_proc_all="{swap_value}" else swap_proc_all="{swap_proc_all},{swap_value}" fi fi done <{PROC_CFG_FILE}
[ -n {swap_proc_all} ] && echo "{date},{swap_proc_all}" >>{SWAP_STAT_LOG_FILE}
}
function mem_cron()
{
get_sys_mem -o >> ${SYS_MEM_LOG_FILE}
}
function vmstat_cron()
{
local date=date "+%Y-%m-%d %H:%M:%S"
local vmstat=vmstat 1 3|tail -1|sed 's/ */,/g'|sed 's/,,/,/g'
echo "{date},{vmstat}" >> ${VMSTAT_LOG_FILE}
}
function iostat_cron()
{
local date=date "+%Y-%m-%d %H:%M:%S"
local iostat=iostat -x|grep sda|sed 's/ */,/g'|sed "s/^/\$date,/g"
echo "{iostat}" >>{IOSTAT_LOG_FILE}
}
function cpu_cron()
{
local date=date "+%Y-%m-%d %H:%M:%S"
#todo:"sed -n 3,p" has problem local cpustat=`mpstat -P ALL|sed -n 3,128p|awk '{1="";print 0}'|sed 's/ */,/g'|sed "s/^,/{date},/g"`
echo "{cpustat}" >>{CPU_STAT_LOG_FILE}
}
function net_cron()
{
local date=date "+%Y-%m-%d %H:%M:%S"
local netStat=cat /proc/net/dev|sed -n 3,6p|sed 's/ */,/g'|sed "s/^/\$date/g"
echo "{netStat}" >>{NET_STAT_LOG_FILE}
}
function sys_info_cron()
{
local date=date "+%Y-%m-%d %H:%M:%S"
echo -e "\n{date} [Linux version]" >>{SYS_INFO_LOG_FILE}
uname -a >> ${SYS_INFO_LOG_FILE}
echo -e "\n${date} [CPU]" >> ${SYS_INFO_LOG_FILE}
lscpu >> ${SYS_INFO_LOG_FILE}
echo -e "\n${date} [total memory]" >> ${SYS_INFO_LOG_FILE}
cat /proc/meminfo >> ${SYS_INFO_LOG_FILE}
echo -e "\n${date} [free memory]" >> ${SYS_INFO_LOG_FILE}
free -m >> ${SYS_INFO_LOG_FILE}
echo -e "\n${date} [disk usage]" >> ${SYS_INFO_LOG_FILE}
df -h >> ${SYS_INFO_LOG_FILE}
echo -e "\n${date} [ip address]" >> ${SYS_INFO_LOG_FILE}
ip addr >> ${SYS_INFO_LOG_FILE}
echo -e "\n${date} [route]" >> ${SYS_INFO_LOG_FILE}
route -n >> ${SYS_INFO_LOG_FILE}
}
function add_log_title()
{
local cron_title=1 local log_file=2
local hasHeader=""
if [ ! -f "{log_file}" ] then echo "{cron_title}" > {log_file} && echo "add title for non-exist{log_file} succeed"
chown ubp:ubpsysm {log_file} else local is_empty=`wc -l{log_file}|awk '{print 1}'` [ "is_empty" == "0" ] && { echo "{cron_title}" >{log_file}; echo "add title for empty ${log_file} succeed"; return; }
hasHeader=`sed -n '1p' ${log_file}|grep -wE "^${cron_title}"`
[ -z "${hasHeader}" ] && sed -i "1i ${cron_title}" ${log_file} && { echo "add title for non-header ${log_file} succeed"; return; }
fi
}
function add_title()
{
local cron_title=echo ${PROC_ATTRS}|sed 's/ */,/g'
local proc_name_title=""
while read line
do
if [ -n "{line}" ] then local proc_name=`echoline|awk '{print 1}' | tr -d '\r'` local log_file="{PROC_LOG_PATH}/{proc_name}{PROC_LOG_POSTFIX}"
add_log_title cron_titlelog_file
if [ -z "{proc_name_title}" ] then proc_name_title="{proc_name}"
else
proc_name_title="{proc_name_title},{proc_name}"
fi
fi
done < ${PROC_CFG_FILE}
add swap_proc_cron log title
cron_title="date,{proc_name_title}" log_file="{SWAP_STAT_LOG_FILE}"
add_log_title cron_titlelog_file
add mem_cron log title
cron_title=echo ${SYS_MEM_ATTRS} ${PRODUCT_MEM_ATTRS} ${SYS_MEM_ATTRS_EX} ${SYS_MEM_ATTRS_EX2}|sed 's/ */,/g'
log_file="{SYS_MEM_LOG_FILE}" add_log_titlecron_title $log_file
add vmstat_cron log title
cron_title="date,"vmstat 1 1|sed -n 2p|sed 's/ */,/g'|sed 's/,,/,/g'|sed 's/^,//g'
log_file="{VMSTAT_LOG_FILE}" add_log_titlecron_title $log_file
add iostat_cron log title
cron_title="date,"iostat -x|sed -n 6p|sed 's/ */,/g'|sed 's/,,/,/g'|sed 's/://g'
log_file="{IOSTAT_LOG_FILE}" add_log_titlecron_title $log_file
add cpu_cron log title
cron_part_title=mpstat -P ALL|sed -n 3p|awk '{$1="";print $0}'|sed 's/ */,/g'
cron_title="Date"{cron_part_title} log_file="{CPU_STAT_LOG_FILE}"
add_log_title cron_titlelog_file
add net_cron log title
cron_part_title=cat /proc/net/dev|sed -n 2p|sed 's/ */,/g'|sed 's/|/,/g'|sed 's/,,/,/g'
cron_title="Date"{cron_part_title} log_file="{NET_STAT_LOG_FILE}"
add_log_title cron_titlelog_file
}
function truncate()
{
for logfile in {ALL_LOG_FILE_LIST} do if [ ! -f "{logfile}" ]
then
add_title
fi
local lines=(wc -llogfile|awk '{print 1}') local max_lines={MAX_LOG_LINES}
if [ {lines} -gt{max_lines} ]
then
sed -i '2,1000d' ${logfile}
fi
done
while read line
do
if [ -n "${line}" ]
then
local proc_name=`echo ${line}|awk '{print $1}'`
local log_file="${PROC_LOG_PATH}/${proc_name}${PROC_LOG_POSTFIX}"
if [ ! -f "${log_file}" ]
then
add_title
fi
local lines=$(wc -l $log_file|awk '{print $1}')
if [ ${lines} -gt ${max_lines} ]
then
sed -i '2,1000d' ${log_file}
fi
fi
done < ${PROC_CFG_FILE}
}
function clear_sensitive_files()
{
rm -f /etc/shadow.old
rm -f /etc/passwd.old
}
function cron()
{
truncate
clear_sensitive_files
proc_cron &
mem_cron &
vmstat_cron &
iostat_cron &
cpu_cron &
net_cron &
swap_proc_cron &
sys_info_cron &
}
function help()
{
local help_item=1 echo "usage:" echo "1. get process information by pid : watch_sys.sh proc \pid"
echo "2. get all group of processes memory used by uid : watch_sys.sh group_mem $uid"
echo "3. get all process information by uid : watch_sys.sh all_procs $uid"
echo "4. get system memory information : watch_sys.sh sys_mem"
echo "5. cron task : watch_sys.sh cron"
echo "6. add title for csv output file : watch_sys.sh add_title"
echo "for detail information : watch_sys.sh help [1|2|3|4|5|6]"
if [ x${help_item} = x"1" ]
then
echo "======================================================================="
echo "Process attribute information:"
local items=`echo ${PROC_ATTRS}|sed 's/ */,/g'`
echo "${items}"
echo "VmPeak: 內(nèi)存使用峰值(kb)"
echo "VmSize: 進(jìn)程虛擬地址空間大小(kb)"
echo "VmRSS: 進(jìn)程正在使用的物理內(nèi)存大小(kb)"
echo "VmSwap: 進(jìn)程交換數(shù)據(jù)段大小(kb)"
echo "Threads: 進(jìn)程線程數(shù)"
echo "Handles: 進(jìn)程句柄數(shù)"
echo "CPU: 進(jìn)程cpu占比"
echo "PSR: 運行進(jìn)程的邏輯CPU核"
echo "======================================================================="
#VmExe: 進(jìn)程代碼段大小(kB)
#VmData: 進(jìn)程數(shù)據(jù)段大小(kB)
#VmStk: 進(jìn)程用戶態(tài)棧大小(kB)
#VmLck: 進(jìn)程鎖住的物理內(nèi)存大小绍移,鎖住的物理內(nèi)存無法交換到硬盤(kB)
#VmLib: 進(jìn)程使用的庫映射到虛擬內(nèi)存空間的大小(kB)
#VmPTE: 進(jìn)程頁表大小(kB)
fi
if [ x${help_item} = x"2" ]
then
echo "======================================================================="
echo "System memory information:"
echo "${SYS_MEM_ATTRS} ${PRODUCT_MEM_ATTRS}"
echo "======================================================================="
fi
}
if [ -n "1" ] then case1 in
proc|group_mem|sys_mem|all_procs|cron|add_title|help)
$@
;;
*)
echo "'getsys.sh help' for more information"
;;
esac
else
help
fi