User Tools

Site Tools


memory

Memory / Swap Troubleshooting

# free -mt

Percent Used/Free:

free -m | grep Mem | awk '{print $3/$2 * 100.0}';
free -m | grep Mem | awk '{print $4/$2 * 100.0}'

FULL MEMORY USAGE

cat /proc/meminfo

cat /proc/meminfo | grep Mem

MEMORY SUMMARY

User/rss:

ps -e -o user,pid,%cpu,%mem,rss,cmd --sort=-rss

vmstat memory summary”

vmstat -s -S M

One liner for ps user/rss and vmstat:

echo -ne '\n';echo "===========================================";ps -eo user,%cpu,%mem,rsz,args|sort -rnk4|awk 'BEGIN {printf "%s\t%s\t%s\t%s\t%s\n","USER","%CPU","%MEM","RSZ","COMMAND"}{printf "%s\t%g'%'\t%g'%'\t%d MB\t%-10s\n",$1,$2,$3,$4/1024,$5}'|head -n30;echo "===========================================";echo -e "\n===========================================";vmstat -s -S M|head -n10;echo "===========================================";

PROCESS MEMORY USAGE

cat /proc/$PID/status

SLABTOP

slabtop

#push c to sort by highest cache user

########################### < Memory Troubleshooting and sar history > ###########################

Lots of memory info in one command:

resize;clear;echo;date;echo "Top 10 Processes by MEM %"; vmstat -a -S m|tail -n1|awk \
'BEGIN {FS=" "}{printf "\nAvail\tActive\tTotal\tPercent Avail\n%sMB\t\
%sMB\t%sMB\t%s\n\n",$4+$5,$6,$4+$5+$6,($4+$5)/($4+$5+$6)*100}';ps -eo \
user,%cpu,%mem,rsz,args|sort -rnk4|awk 'BEGIN {printf "%8s %6s %6s \
%8s     %-10s\n","USER","%CPU","%MEM","RSZ","COMMAND"}{printf "%8s %6s \
%6s %8s MB  %-10s\n",$1,$2,$3,$4/1024,$5}'|head -n10; echo ""; echo "== \
Last Half Hour ==";echo; sar -r|head -n3; sar -r|tail -n4;echo; sar -B|\
head -n3; sar -B|tail -n4;echo;echo "== Current 2 Second Intervals ==";\
echo;sar -r 2 5;echo;sar -B 2 5

TOP 10 MEMORY USERS

ps auxk-rss | head -11 | cut -c1-120

MEMORY % (RHEL 4)

sar -r | tail -n21 | head -n20 | awk '{ printf "%s -- %3.2f%%\n",$1,($3-$5-$6)*100/($3+$2)}'

DAY MEMORY AVERAGE

sar -r | grep Average: | awk '{print $1" \t"$4"%"}'

SAR DAILY

for i in `ls -rt /var/log/sa/ | grep -E "sa[0-9][0-9]"`; do echo -ne "$i -- "; sar -r -f /var/log/sa/$i | grep -Ev "Linux|Average|RESTART|kbmemfree|^$" | awk '{ printf "%3.2f\n",($4-$6-$7)*100/($3+$4)}' | awk '{sum+=$1 } END { printf "Average = %3.2f%%\n",sum/NR}'; done

SAR HOURLY

sar -r |\
awk '
{
    if (NR > 3 && $3 != "kbmemfree" && $3 != "") {
        if ($1 != "Average:") {
            kbmemfree = $3
            kbmemused = $4
            kbbuffers = $6
            kbcached = $7
        }
        else {
            kbmemfree = $2
            kbmemused = $3
            kbbuffers = $5
            kbcached = $6
        }

        kbfree_plus_bufcache = kbmemfree + kbbuffers + kbcached
        kbused_minus_bufcache = kbmemused - kbbuffers - kbcached

        printf("%s %15d %15d\n", $0, kbfree_plus_bufcache, kbused_minus_bufcache)
    } else if ($3 == "kbmemfree") {
        printf("%s %15s %15s\n", $0, "kbfree+bufcache", "kbused-bufcache")
    } else {
        printf("%s\n", $0)
    }
}'

MONTHLY AVERAGES

for i in `ls /var/log/sa/ | egrep 'sa[0-9][0-9]'` ; do sar -f /var/log/sa/$i -r | grep Average | awk '{print $1" "$4"%"}' ; done
sar -r | grep -v Average | awk '{print $1" "$2" \t"$5"%"}';sar -r | grep Average: | awk '{print $1" \t"$4"%"}'

BETTER FREE REPORT

date ; free -mt ; free -m|awk 'NR==2{print "The total % of "Mem" Used: "($3*100)/$2"%"}'

DENTRY INODE CACHE

head -2 /proc/slabinfo; cat /proc/slabinfo  | egrep dentry\|inode

CACHE CLEARING

sync; echo "2" > /proc/sys/vm/drop_caches; sleep 10; echo "0" > /proc/sys/vm/drop_caches

MORE CACHE CLEARING & DETAILS

sync && echo 1 > /proc/sys/vm/drop_caches

clean up memory of unnecessary things (Kernerl 2.6.16 or newer)

run sync first to flush useful things out to disk!!!

To free pagecache:

echo 1 > /proc/sys/vm/drop_caches

To free dentries and inodes:

echo 2 > /proc/sys/vm/drop_caches

To free pagecache, dentries and inodes:

echo 3 > /proc/sys/vm/drop_caches

APACHE MEMORY USAGE

ps -eo rsz,args | grep httpd | awk ' { SUM += $1 } END { print "Memory used by Apache = "SUM/1024 " Megs" "\nNumber of process runing = " NR "\nAverage of each process mem usage = " SUM/1024/NR " Megs"} '

JAVA MEMORY USAGE

ps afux | grep java | grep -Eo "\-Xmx[0-9]+[m|g] "

More Memory Investigation

free -mt
vmstat -a -S m
ps auxk-rss |head -11
ps auxk-rss |head -11 | awk '{print $1, $4, $11}'

#Memory %
sar -r | grep -v Average | awk '{print $1" "$2" \t%"$5}'
#Memory Average %
sar -r | grep Average: | awk '{print $1" \t"$4"%"}'

#Monthly Averges
for i in `ls /var/log/sa/ | egrep 'sa[0-9][0-9]'` ; do sar -f /var/log/sa/$i -r | grep Average | awk '{print $1" "$4"%"}' ; done

sar -r | grep -v Average | awk '{print $1" "$2" \t"$5"%"}';sar -r | grep Average: | awk '{print $1" \t"$4"%"}'

date ; free -mt ; free -m|awk 'NR==2{print "The total % of "Mem" Used: "($3*100)/$2"%"}'

#Nimbus Memory %
(unset LANG ;sar -r) |awk '$3~/[0-9]/{total=$3+$2; usedbc=$3-($5+$6); pc_used=(100*usedbc)/total;print $0,pc_used} $3!~/[0-9]/{print $0}' | awk '{print $1" - "$11"%"}'

#Nimbus Memory % (RHEL 4)
sar -r | tail -n21 | head -n20 | awk '{ printf "%s -- %3.2f%%\n",$1,($3-$5-$6)*100/($3+$2)}'


//Apache memory://
 ps -eo rsz,args | grep httpd | awk ' { SUM += $1 } END { print "Memory used by Apache = "SUM/1024 " Megs" "\nNumber of process runing = " NR "\nAverage of each process mem usage = " SUM/1024/NR " Megs"} '


//low memory://
resize;clear;echo;date;echo "Top 10 Processes by MEM %"; vmstat -a -S m|tail -n1|awk \
'BEGIN {FS=" "}{printf "\nAvail\tActive\tTotal\tPercent Avail\n%sMB\t\
%sMB\t%sMB\t%s\n\n",$4+$5,$6,$4+$5+$6,($4+$5)/($4+$5+$6)*100}';ps -eo \
user,%cpu,%mem,rsz,args|sort -rnk4|awk 'BEGIN {printf "%8s %6s %6s \
%8s     %-10s\n","USER","%CPU","%MEM","RSZ","COMMAND"}{printf "%8s %6s \
%6s %8s MB  %-10s\n",$1,$2,$3,$4/1024,$5}'|head -n10; echo ""; echo "== \
Last Half Hour ==";echo; sar -r|head -n3; sar -r|tail -n4;echo; sar -B|\
head -n3; sar -B|tail -n4;echo;echo "== Current 2 Second Intervals ==";\
echo;sar -r 2 5;echo;sar -B 2 5

//Something with memory and sar//
for i in `ls -rt /var/log/sa/ | grep -E "sa[0-9][0-9]"`; do echo -ne "$i -- "; sar -r -f /var/log/sa/$i | grep -Ev "Linux|Average|RESTART|kbmemfree|^$" | awk '{ printf "%3.2f\n",($4-$6-$7)*100/($3+$4)}' | awk '{sum+=$1 } END { printf "Average = %3.2f%%\n",sum/NR}'; done

Show whats using SWAP:

sh swapusage.sh | sort -n -k1 | tac | head -n10
#!/bin/bash
#
# show swap used by processes
#
(for PROCESS in /proc/*/; do
  swapused=$(awk 'BEGIN { total = 0 } /^Swap:[[:blank:]]*[1-9]/ { total = total + $2 } END { print total }' ${PROCESS}/smaps 2>/dev/null || echo 0)
  if [ $swapused -gt 0 ]; then
    /bin/echo -e "${swapused}k\t$(cat ${PROCESS}/cmdline)"
  fi
done ) | sort -nr
#!/bin/bash
# Get current swap usage for all running processes
# Erik Ljungstrom 27/05/2011
SUM=0
OVERALL=0
for DIR in `find /proc/ -maxdepth 1 -type d | egrep "^/proc/[0-9]"` ; do
PID=`echo $DIR | cut -d / -f 3`
PROGNAME=`ps -p $PID -o comm --no-headers`
for SWAP in `grep Swap $DIR/smaps 2>/dev/null| awk '{ print $2 }'`
do
let SUM=$SUM+$SWAP
done
echo "PID=$PID - Swap used: $SUM - ($PROGNAME )"
let OVERALL=$OVERALL+$SUM
SUM=0

done
echo "Overall swap used: $OVERALL"

Pipe thru this for only swap using procs:

| egrep -v "Swap used: 0" |sort -n -k 5

Swap one liner:

SUM=0; OVERALL=0; for DIR in `find /proc/ -maxdepth 1 -type d | egrep "^/proc/[0-9]"` ; do PID=`echo $DIR | cut -d / -f 3`; PROGNAME=`ps -p $PID -o comm --no-headers`; for SWAP in `grep Swap $DIR/smaps 2>/dev/null| awk '{ print $2 }'`; do let SUM=$SUM+$SWAP; done; echo "PID=$PID - Swap used: $SUM - ($PROGNAME )"; SUM=0; done; echo "Overall swap used: $OVERALL"; 

Works in progress/broken:

sar -r | grep -v Average | awk '{print $1" "$2" \t%"$5}'
(unset LANG ;sar -r) |awk '$3~/[0-9]/{total=$3+$2; usedbc=$3-($5+$6); pc_used=(100*usedbc)/total;print $0,pc_used} $3!~/[0-9]/{print $0}' | awk '{print $1" - "$11"%"}'
memory.txt · Last modified: 2016/07/18 09:32 by vinny