diff options
Diffstat (limited to 'trafsum')
-rw-r--r-- | trafsum/README | 20 | ||||
-rw-r--r-- | trafsum/traflog.awk | 45 | ||||
-rwxr-xr-x | trafsum/trafsum | 160 | ||||
-rw-r--r-- | trafsum/trafsum.conf | 11 | ||||
-rw-r--r-- | trafsum/trafsum.local | 50 | ||||
-rw-r--r-- | trafsum/translate | 7 |
6 files changed, 293 insertions, 0 deletions
diff --git a/trafsum/README b/trafsum/README new file mode 100644 index 0000000..cf7f56a --- /dev/null +++ b/trafsum/README @@ -0,0 +1,20 @@ +This is trafsum: a traffic data summator. It processes the information +from one or several log files and stores the data in MySQL database. The +format of log files is as follows: + +# <---- Slice time ----> IP-Address Inbytes Outbytes +Mon Jun 4 03:30:02 2001 213.130.7.7 4012128 7627102 + +To install: + + 1. Copy the entire directory somewhere where you'd like it to be installed; + 2. Copy the file trafsum.conf to /etc and edit it to your liking. It + should have access permissions of 600. + 3. If necessary, you may wish to edit <instdir>/translate. The necessary + instructons are found in the comment at the start of the file; + 4. Add the script to your crontab: + + 20 0 * * * /usr/local/trafsum/trafsum >/var/log/trafsum.err 2>&1 + + 5. Take cover and wait for the script to work. + diff --git a/trafsum/traflog.awk b/trafsum/traflog.awk new file mode 100644 index 0000000..7792fb9 --- /dev/null +++ b/trafsum/traflog.awk @@ -0,0 +1,45 @@ +# $Id: traflog.awk,v 1.1 2005/08/04 07:15:38 gray Exp $ +# Copyright 2000, 2005 Sergey Poznyakoff +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +#Field assignment +# 1 2 3 4 5 6 7 8 +#Sun Jun 20 00:04:06 1999 208.244.48.32 30 1736 + +BEGIN { + divisor = 1048576 + maxtraf = 4294960000 +} + +NF==8 { + # Sanity check + if (0+$7 < 0 || 0+$8 < 0 || 0+$7 > maxtraf || 0+$8 > maxtraf) { + print NR ": Skipped (" $7 " " $8 ")" >"/dev/stderr" + next + } else { + inbytes[$6] += 0+$7 + outbytes[$6] += 0+$8 + } +} + +END { + for (ip in inbytes) { + print ip " " inbytes[ip] / divisor " " outbytes[ip] / divisor + } +} + + diff --git a/trafsum/trafsum b/trafsum/trafsum new file mode 100755 index 0000000..10437bd --- /dev/null +++ b/trafsum/trafsum @@ -0,0 +1,160 @@ +#! /bin/sh +# Copyright 2000, 2005 Sergey Poznyakoff +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin +export PATH + +BASEDIR=`dirname $0` +SED=/tmp/trafsum.$$.sed +TEMP=/tmp/trafsum.$$ +TERR=/tmp/trafsum.err.$$ +CONF=/etc/trafsum.conf +SUMMARY=/tmp/trafsum.summary +LOG=/var/log/ipstat/trafsum.log +SQLLOG=/var/log/ipstat/trafsum.sql +DATE=`date +%y%m%d%n -d yesterday` +STDERR=/var/log/ipstat/trafsum.err +HISTORY=/var/log/ipstat/history + +log() { + echo "$*" | tee -a $TEMPLOG >> $LOG +} + +## Determine yesterday's date + +yesterday() { + datestr=`date +%Y-%m-%d -d "-1 day"` +} + +lookup() { + NAME=`nslookup $1 2>/dev/null | sed -ne 's/Name: *\(.*\)/\1/p'` + if [ "$NAME" != "" ]; then + cp $BASEDIR/namelist $BASEDIR/namelist.b + if fgrep "$1:" $BASEDIR/namelist 2>/dev/null ; then + sed -e "s/$1:.*$/$1: $NAME/" $BASEDIR/namelist.b > $BASEDIR/namelist + else + echo "$1: $NAME" >> $BASEDIR/namelist + fi + log "Cached DNS entry $1: $NAME" + else + NAME=$1 + fi +} + +########################################### +### Main +# + +. $CONF + +trap 'rm -f $SED $TEMP $TEMPLOG $SUMMARY' 0 1 2 3 15 + +if [ x"$SOURCE_FILES" = x"" ]; then + log "Configuration error: no source files specified" + exit 1 +fi +cat /dev/null > $SUMMARY +cat /dev/null > $STDERR + +if [ ! -f $LOG ] ; then + cat /dev/null > $LOG +fi + +cat $SOURCE_FILES > $SUMMARY + +# Get yesterday's date into $datestr +yesterday + +echo -e "\ndate: $datestr" + +## Process statistics data +# +if [ -r $BASEDIR/translate ]; then + sed -ne 's%\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\):[^0-9]*\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\).*%s/\1\\.\2\\.\3\\.\4/\5\\.\6\\.\7\\.\8/%p' $BASEDIR/translate > $SED +else + rm -f $SED # Make sure it does not exist +fi + +MYSQL_OPTIONS="-A " +if [ "$SERVER" ]; then + MYSQL_OPTIONS="$MYSQL_OPTIONS -h$SERVER" +fi + +if [ "$PORT" ]; then + MYSQL_OPTIONS="$MYSQL_OPTIONS -P$PORT" +fi + +if [ "$MYSQL_USER" ]; then + MYSQL_OPTIONS="$MYSQL_OPTIONS -u$MYSQL_USER" +fi + +if [ "$MYSQL_PASSWORD" ]; then + MYSQL_OPTIONS="$MYSQL_OPTIONS -p$MYSQL_PASSWORD" +fi +MYSQL_OPTIONS="$MYSQL_OPTIONS $DATABASE" + +cat $SUMMARY | +( + if [ -r $SED ]; then + sed -f $SED + else + cat - + fi | \ + awk -f $BASEDIR/traflog.awk 2>$TERR | + while read IPADDR INBYTES OUTBYTES + do + NAME=`fgrep "$IPADDR:" $BASEDIR/namelist | sed -e 's/.*: *\(.*\)/\1/' 2>/dev/null` + [ "$NAME" = "" ] && lookup $IPADDR + echo -e "$IPADDR\t$NAME\t$INBYTES\t\t$OUTBYTES" >> $TEMP + echo "INSERT INTO $TABLE VALUES ('$IPADDR','$datestr',$INBYTES,$OUTBYTES);" + done +) | + tee $SQLLOG | + mysql $MYSQL_OPTIONS + +[ x"$ADDRESSLIST" != x"" ] && \ +( + if [ -s $TERR ]; then + echo "WARNING: traflog.awk reported errors. They are:" + echo "===============================================" + cat $TERR + echo "===============================================" + fi + + sort -t'.' -n +0 -1 +1 -2 +2 -3 +3 -4 $TEMP | \ + awk -v DATE=$datestr 'BEGIN { + print "I/O traffic summary for " DATE + print + printf "%-17.17s%24.24s%15s%15s\n", "IP", "Name/address", "Incoming", "Outgoing" + } + { printf "%-17.17s%24.24s%15s%15s\n", $1, $2, $3, $4 }' \ +) | mail -s "traffic statistics" $ADDRESSLIST + +cat $SUMMARY | gzip - > $HISTORY/${DATE}.gz + +if [ -x $BASEDIR/trafsum.local ]; then + sh $BASEDIR/trafsum.local $SUMMARY +fi + +rm -f $SED $TEMP $TEMPLOG $SUMMARY $TERR +for i in $SOURCE_FILES +do + cat /dev/null > $i +done + + diff --git a/trafsum/trafsum.conf b/trafsum/trafsum.conf new file mode 100644 index 0000000..dd1df10 --- /dev/null +++ b/trafsum/trafsum.conf @@ -0,0 +1,11 @@ +# Config file for traffic accounter + +SERVER=#<MYSQL server name> +PORT=#<MYSQL port> +DATABASE=#<Database name> +TABLE=#<Table name> +MYSQL_USER= +MYSQL_PASSWORD= +SOURCE_FILES="" # Ws-delimited list of files to be analyzed +ADDRESSLIST="" # Ws-delimited list of email addresses the report should be sent + # to diff --git a/trafsum/trafsum.local b/trafsum/trafsum.local new file mode 100644 index 0000000..d88f979 --- /dev/null +++ b/trafsum/trafsum.local @@ -0,0 +1,50 @@ +#! /bin/sh +# This is an example of trafsum.local script. +# Copyright 2000 Sergey Poznyakoff +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +FILE=$1 +DATE=`date --date '1 day ago'` +( + cat - <<EOF +From: Trafsum <noreturn@some.dom.ain> +Subject: Your statistics for $DATE + +********************************************************************* +* THE RETURN ADDRESSES ON THIS LETTER HAVE BEEN SET TO PREVENT MAIL * +* LOOPS IN THE EVENT YOU ARE RUNNING SOFTWARE WHICH AUTO-REPLIES TO * +* INBOUND MAIL. WE WILL NOT SEE ANY REPLY SENT TO THIS LETTER * +********************************************************************* + +Dear customer, + +Please find below the statistics of your traffic consumption for $DATE + +--------------------------------------------------------------------- +EOF + + grep 213.130.3.56 $FILE + + cat - <<EOF +--------------------------------------------------------------------- + +Kind regards, +Trafsum, +Statistics robot +EOF + ) | sendmail some.addr@some.domain + diff --git a/trafsum/translate b/trafsum/translate new file mode 100644 index 0000000..a163d1d --- /dev/null +++ b/trafsum/translate @@ -0,0 +1,7 @@ +# This file is used to translate IP addresses for those customers who +# have more than one IP assigned +# Syntax is: +# <from-ip-addr>:<to-ip-addr> +# In the example below all traffic for 213.130.4.43 will be summed up +# with that for 213.130.4.40: +#213.130.4.43:213.130.4.40 Comment |