Skip to content
This repository has been archived by the owner. It is now read-only.

Commit

Permalink
Give more detail in the top 10 tables in the monthly/yearly stats
Browse files Browse the repository at this point in the history
  • Loading branch information
Rhys Smith committed Oct 29, 2016
1 parent 4f24cd5 commit 57807e6
Showing 1 changed file with 94 additions and 8 deletions.
102 changes: 94 additions & 8 deletions utilities/stats-generate.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,11 +372,56 @@ else
mdaggrmaxqueriesperipfull="0"
fi

# Top 10 downloaders and how many downloads / total data shipped (full downloads only) (don't count these when doing daily stats)
# Don't count these when doing daily stats
if [[ "$timeperiod" != "day" ]]; then
mdaggrtoptenbycount=$(grep $apachesearchterm $logslocation/md/md1/metadata.uou-access_log* $logslocation/md/md2/metadata.uou-access_log* $logslocation/md/md3/metadata.uou-access_log* | grep -Ev "(Sensu-HTTP-Check|dummy|check_http|Balancer)" | grep ".xml" | grep -v 404 | grep "\" 200" | grep "GET" | grep -v 193.63.72.83 | grep -v 194.83.7.211 | cut -f 2 -d ":" | cut -f 1 -d " " | sort | uniq -c | sort -nr | head -10)

# Top 10 downloaders and how many downloads / total data shipped (full downloads only)
if [[ "$timeperiod" != "day" ]]; then
mdaggrtoptenipsbycount=$(grep $apachesearchterm $logslocation/md/md1/metadata.uou-access_log* $logslocation/md/md2/metadata.uou-access_log* $logslocation/md/md3/metadata.uou-access_log* | grep -Ev "(Sensu-HTTP-Check|dummy|check_http|Balancer)" | grep ".xml" | grep -v 404 | grep "\" 200" | grep "GET" | grep -v 193.63.72.83 | grep -v 194.83.7.211 | cut -f 2 -d ":" | cut -f 1 -d " " | sort | uniq -c | sort -nr | head -10)
fi

#
# Manipute results of the top 10
#

# Blank the output we're going to set
mdaggrtoptenipsbycountdetailed=""

i=1
IFS=$'\n'
for line in $mdaggrtoptenipsbycount
do
# Parse the line
count=$(echo $line | awk '{print $1}')
ipaddr=$(echo $line | awk '{print $2'})

# Make count friendly
countfriendly=$(echo $count | awk '{ printf ("%'"'"'d\n", $0) }')

# Figure out total traffic shipped to this IP
totaldataforthisip=$(grep $apachesearchterm $logslocation/md/md1/metadata.uou-access_log* $logslocation/md/md2/metadata.uou-access_log* $logslocation/md/md3/metadata.uou-access_log* | grep -Ev "(Sensu-HTTP-Check|dummy|check_http|Balancer)" | grep ".xml" | grep -v 404 | grep "\" 200" | grep "GET" | grep $ipaddr | cut -f 10 -d " " | grep -v - | awk '{sum+=$1} END {print sum}')
if [[ "$totaldataforthisip" -gt "0" ]]; then
totaldataforthisiphr=$(bytestohr $totaldataforthisip)
else
totaldataforthisiphr="0 B"
fi

# Get Reverse DNS for this IP address
rdnsforthisip=$(dig +noall +answer -x $ipaddr | awk '{print $5}')
if [[ -z $rdnsforthisip ]]; then
rdnsforthisip="No RDNS available"
fi

# Add to the output
mdaggrtoptenipsbycountdetailed+="$i: $countfriendly D/Ls ($totaldataforthisiphr) from $ipaddr ($rdnsforthisip)\n"

# Increment the count, and blank the rdns response
i=$((i+1))
rdnsforthisip=""
done
fi


# =====
# MDQ stats
# =====
Expand Down Expand Up @@ -455,7 +500,7 @@ mdqtotalbytes=$(grep $apachesearchterm $logslocation/md/md1/mdq.uou-access_log*
if [[ "$mdqtotalbytes" -gt "0" ]]; then
mdqtotalhr=$(bytestohr $mdqtotalbytes)
else
mdqtotalgb="0 B"
mdqtotalhr="0 B"
fi

# Min queries per IP
Expand Down Expand Up @@ -483,6 +528,47 @@ if [[ "$timeperiod" != "day" ]]; then
# Top 10 downloaders and how many downloads / total data shipped
mdqtoptenipsbycount=$(grep $apachesearchterm $logslocation/md/md1/mdq.uou-access_log* $logslocation/md/md2/mdq.uou-access_log* $logslocation/md/md3/mdq.uou-access_log* | grep -Ev "(Sensu-HTTP-Check|dummy|check_http|Balancer)" | grep -v 193.63.72.83 | grep -v 194.83.7.211 | grep "/entities" | grep -v 404 | grep -v "/entities/ " | cut -f 2 -d ":" | cut -f 1 -d " " | sort | uniq -c | sort -nr | head -10)

#
# Manipute results of the top 10
#

# Blank the output we're going to set
mdqtoptenipsbycountdetailed=""

i=1
IFS=$'\n'
for line in $mdqtoptenipsbycount
do
# Parse the line
count=$(echo $line | awk '{print $1}')
ipaddr=$(echo $line | awk '{print $2'})

# Make count friendly
countfriendly=$(echo $count | awk '{ printf ("%'"'"'d\n", $0) }')

# Figure out total traffic shipped to this IP
totaldataforthisip=$(grep $apachesearchterm $logslocation/md/md1/mdq.uou-access_log* $logslocation/md/md2/mdq.uou-access_log* $logslocation/md/md3/mdq.uou-access_log* | grep -Ev "(Sensu-HTTP-Check|dummy|check_http|Balancer)" | grep "/entities/" | grep -v 404 | grep "\" 200" | grep $ipaddr | cut -f 10 -d " " | grep -v - | awk '{sum+=$1} END {print sum}')
if [[ "$totaldataforthisip" -gt "0" ]]; then
totaldataforthisiphr=$(bytestohr $totaldataforthisip)
else
totaldataforthisiphr="0 B"
fi

# Get Reverse DNS for this IP address
rdnsforthisip=$(dig +noall +answer -x $ipaddr | awk '{print $5}')
if [[ -z $rdnsforthisip ]]; then
rdnsforthisip="No RDNS available"
fi

# Add to the output
mdqtoptenipsbycountdetailed+="$i: $countfriendly D/Ls ($totaldataforthisiphr) from $ipaddr ($rdnsforthisip)\n"

# Increment the count, and blank the rdns response
i=$((i+1))
rdnsforthisip=""
done


# Top 10 queries and how many downloads / total data shipped
mdqtoptenqueriesbycount=$(grep $apachesearchterm $logslocation/md/md1/mdq.uou-access_log* $logslocation/md/md2/mdq.uou-access_log* $logslocation/md/md3/mdq.uou-access_log* | grep -Ev "(Sensu-HTTP-Check|dummy|check_http|Balancer)" | grep -v 193.63.72.83 | grep -v 194.83.7.211 | grep /entities/ | grep -v 404 | grep -v "/entities/ " | awk '{print $7}' | cut -f 3 -d "/" | sed "s@+@ @g;s@%@\\\\x@g" | xargs -0 printf "%b" | sort | uniq -c | sort -nr | head -10)
fi
Expand Down Expand Up @@ -612,7 +698,7 @@ else
msg+="-> * ukfederation-cdsall.xml = $mdaggrcdsallcountfriendly requests ($mdaggrcdsallpc% of total)\n"
msg+="-> * ukfederation-wayf.xml = $mdaggrwayfcountfriendly requests ($mdaggrwayfpc% of total)\n"
msg+="\nTop 10 downloaders (full downloads only):\n"
msg+="$mdaggrtoptenbycount\n"
msg+="$mdaggrtoptenipsbycountdetailed\n"
msg+="\n-----\n"
msg+="MDQ:\n"
msg+="-> $mdqcountfriendly requests* from $mdquniqueipfriendly clients, $mdqtotalhr shipped.\n"
Expand All @@ -622,7 +708,7 @@ else
msg+="-> $mdqminqueriesperip min/$mdqavgqueriesperip avg/$mdqmaxqueriesperip max queries per querying IP\n"
msg+="-> $mdqcountallentities queries for collection of all entities\n"
msg+="\nTop 10 queryers:\n"
msg+="$mdqtoptenipsbycount\n"
msg+="$mdqtoptenipsbycountdetailed\n"
msg+="\nTop 10 entities queried for:\n"
msg+="$mdqtoptenqueriesbycount\n"
msg+="\n-----\n"
Expand All @@ -637,14 +723,14 @@ else
msg+="\n-----\n"
msg+="Test IdP usage:\n"
msg+="-> $testidplogincount logins to $testidpspcount SPs.\n"
msg+="\nLogins per test user:\n"
msg+="\n-> Logins per test user:\n"
msg+="$testidplogincountbyuser\n"
msg+="\nTop 10 SPs logged into:\n"
msg+="\n-> Top 10 SPs logged into:\n"
msg+="$testidptoptenspsbycount\n"
msg+="\n-----\n"
msg+="Test SP usage:\n"
msg+="-> $testsplogincount logins from $testspidpcount IdPs.\n"
msg+="\nTop 10 IdPs logged in from:\n"
msg+="\n-> Top 10 IdPs logged in from:\n"
msg+="$testsptoptenidpsbycount\n"
msg+="\n-----"
fi
Expand Down

0 comments on commit 57807e6

Please sign in to comment.