aboutsummaryrefslogtreecommitdiff
path: root/trafsum
diff options
context:
space:
mode:
Diffstat (limited to 'trafsum')
-rw-r--r--trafsum/README20
-rw-r--r--trafsum/traflog.awk45
-rwxr-xr-xtrafsum/trafsum160
-rw-r--r--trafsum/trafsum.conf11
-rw-r--r--trafsum/trafsum.local50
-rw-r--r--trafsum/translate7
6 files changed, 293 insertions, 0 deletions
diff --git a/trafsum/README b/trafsum/README
new file mode 100644
index 0000000..cf7f56a
--- /dev/null
+++ b/trafsum/README
@@ -0,0 +1,20 @@
+This is trafsum: a traffic data summator. It processes the information
+from one or several log files and stores the data in MySQL database. The
+format of log files is as follows:
+
+# <---- Slice time ----> IP-Address Inbytes Outbytes
+Mon Jun 4 03:30:02 2001 213.130.7.7 4012128 7627102
+
+To install:
+
+ 1. Copy the entire directory somewhere where you'd like it to be installed;
+ 2. Copy the file trafsum.conf to /etc and edit it to your liking. It
+ should have access permissions of 600.
+ 3. If necessary, you may wish to edit <instdir>/translate. The necessary
+ instructons are found in the comment at the start of the file;
+ 4. Add the script to your crontab:
+
+ 20 0 * * * /usr/local/trafsum/trafsum >/var/log/trafsum.err 2>&1
+
+ 5. Take cover and wait for the script to work.
+
diff --git a/trafsum/traflog.awk b/trafsum/traflog.awk
new file mode 100644
index 0000000..7792fb9
--- /dev/null
+++ b/trafsum/traflog.awk
@@ -0,0 +1,45 @@
+# $Id: traflog.awk,v 1.1 2005/08/04 07:15:38 gray Exp $
+# Copyright 2000, 2005 Sergey Poznyakoff
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA.
+
+#Field assignment
+# 1 2 3 4 5 6 7 8
+#Sun Jun 20 00:04:06 1999 208.244.48.32 30 1736
+
+BEGIN {
+ divisor = 1048576
+ maxtraf = 4294960000
+}
+
+NF==8 {
+ # Sanity check
+ if (0+$7 < 0 || 0+$8 < 0 || 0+$7 > maxtraf || 0+$8 > maxtraf) {
+ print NR ": Skipped (" $7 " " $8 ")" >"/dev/stderr"
+ next
+ } else {
+ inbytes[$6] += 0+$7
+ outbytes[$6] += 0+$8
+ }
+}
+
+END {
+ for (ip in inbytes) {
+ print ip " " inbytes[ip] / divisor " " outbytes[ip] / divisor
+ }
+}
+
+
diff --git a/trafsum/trafsum b/trafsum/trafsum
new file mode 100755
index 0000000..10437bd
--- /dev/null
+++ b/trafsum/trafsum
@@ -0,0 +1,160 @@
+#! /bin/sh
+# Copyright 2000, 2005 Sergey Poznyakoff
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA.
+
+PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
+export PATH
+
+BASEDIR=`dirname $0`
+SED=/tmp/trafsum.$$.sed
+TEMP=/tmp/trafsum.$$
+TERR=/tmp/trafsum.err.$$
+CONF=/etc/trafsum.conf
+SUMMARY=/tmp/trafsum.summary
+LOG=/var/log/ipstat/trafsum.log
+SQLLOG=/var/log/ipstat/trafsum.sql
+DATE=`date +%y%m%d%n -d yesterday`
+STDERR=/var/log/ipstat/trafsum.err
+HISTORY=/var/log/ipstat/history
+
+log() {
+ echo "$*" | tee -a $TEMPLOG >> $LOG
+}
+
+## Determine yesterday's date
+
+yesterday() {
+ datestr=`date +%Y-%m-%d -d "-1 day"`
+}
+
+lookup() {
+ NAME=`nslookup $1 2>/dev/null | sed -ne 's/Name: *\(.*\)/\1/p'`
+ if [ "$NAME" != "" ]; then
+ cp $BASEDIR/namelist $BASEDIR/namelist.b
+ if fgrep "$1:" $BASEDIR/namelist 2>/dev/null ; then
+ sed -e "s/$1:.*$/$1: $NAME/" $BASEDIR/namelist.b > $BASEDIR/namelist
+ else
+ echo "$1: $NAME" >> $BASEDIR/namelist
+ fi
+ log "Cached DNS entry $1: $NAME"
+ else
+ NAME=$1
+ fi
+}
+
+###########################################
+### Main
+#
+
+. $CONF
+
+trap 'rm -f $SED $TEMP $TEMPLOG $SUMMARY' 0 1 2 3 15
+
+if [ x"$SOURCE_FILES" = x"" ]; then
+ log "Configuration error: no source files specified"
+ exit 1
+fi
+cat /dev/null > $SUMMARY
+cat /dev/null > $STDERR
+
+if [ ! -f $LOG ] ; then
+ cat /dev/null > $LOG
+fi
+
+cat $SOURCE_FILES > $SUMMARY
+
+# Get yesterday's date into $datestr
+yesterday
+
+echo -e "\ndate: $datestr"
+
+## Process statistics data
+#
+if [ -r $BASEDIR/translate ]; then
+ sed -ne 's%\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\):[^0-9]*\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\).*%s/\1\\.\2\\.\3\\.\4/\5\\.\6\\.\7\\.\8/%p' $BASEDIR/translate > $SED
+else
+ rm -f $SED # Make sure it does not exist
+fi
+
+MYSQL_OPTIONS="-A "
+if [ "$SERVER" ]; then
+ MYSQL_OPTIONS="$MYSQL_OPTIONS -h$SERVER"
+fi
+
+if [ "$PORT" ]; then
+ MYSQL_OPTIONS="$MYSQL_OPTIONS -P$PORT"
+fi
+
+if [ "$MYSQL_USER" ]; then
+ MYSQL_OPTIONS="$MYSQL_OPTIONS -u$MYSQL_USER"
+fi
+
+if [ "$MYSQL_PASSWORD" ]; then
+ MYSQL_OPTIONS="$MYSQL_OPTIONS -p$MYSQL_PASSWORD"
+fi
+MYSQL_OPTIONS="$MYSQL_OPTIONS $DATABASE"
+
+cat $SUMMARY |
+(
+ if [ -r $SED ]; then
+ sed -f $SED
+ else
+ cat -
+ fi | \
+ awk -f $BASEDIR/traflog.awk 2>$TERR |
+ while read IPADDR INBYTES OUTBYTES
+ do
+ NAME=`fgrep "$IPADDR:" $BASEDIR/namelist | sed -e 's/.*: *\(.*\)/\1/' 2>/dev/null`
+ [ "$NAME" = "" ] && lookup $IPADDR
+ echo -e "$IPADDR\t$NAME\t$INBYTES\t\t$OUTBYTES" >> $TEMP
+ echo "INSERT INTO $TABLE VALUES ('$IPADDR','$datestr',$INBYTES,$OUTBYTES);"
+ done
+) |
+ tee $SQLLOG |
+ mysql $MYSQL_OPTIONS
+
+[ x"$ADDRESSLIST" != x"" ] && \
+(
+ if [ -s $TERR ]; then
+ echo "WARNING: traflog.awk reported errors. They are:"
+ echo "==============================================="
+ cat $TERR
+ echo "==============================================="
+ fi
+
+ sort -t'.' -n +0 -1 +1 -2 +2 -3 +3 -4 $TEMP | \
+ awk -v DATE=$datestr 'BEGIN {
+ print "I/O traffic summary for " DATE
+ print
+ printf "%-17.17s%24.24s%15s%15s\n", "IP", "Name/address", "Incoming", "Outgoing"
+ }
+ { printf "%-17.17s%24.24s%15s%15s\n", $1, $2, $3, $4 }' \
+) | mail -s "traffic statistics" $ADDRESSLIST
+
+cat $SUMMARY | gzip - > $HISTORY/${DATE}.gz
+
+if [ -x $BASEDIR/trafsum.local ]; then
+ sh $BASEDIR/trafsum.local $SUMMARY
+fi
+
+rm -f $SED $TEMP $TEMPLOG $SUMMARY $TERR
+for i in $SOURCE_FILES
+do
+ cat /dev/null > $i
+done
+
+
diff --git a/trafsum/trafsum.conf b/trafsum/trafsum.conf
new file mode 100644
index 0000000..dd1df10
--- /dev/null
+++ b/trafsum/trafsum.conf
@@ -0,0 +1,11 @@
+# Config file for traffic accounter
+
+SERVER=#<MYSQL server name>
+PORT=#<MYSQL port>
+DATABASE=#<Database name>
+TABLE=#<Table name>
+MYSQL_USER=
+MYSQL_PASSWORD=
+SOURCE_FILES="" # Ws-delimited list of files to be analyzed
+ADDRESSLIST="" # Ws-delimited list of email addresses the report should be sent
+ # to
diff --git a/trafsum/trafsum.local b/trafsum/trafsum.local
new file mode 100644
index 0000000..d88f979
--- /dev/null
+++ b/trafsum/trafsum.local
@@ -0,0 +1,50 @@
+#! /bin/sh
+# This is an example of trafsum.local script.
+# Copyright 2000 Sergey Poznyakoff
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA.
+
+FILE=$1
+DATE=`date --date '1 day ago'`
+(
+ cat - <<EOF
+From: Trafsum <noreturn@some.dom.ain>
+Subject: Your statistics for $DATE
+
+*********************************************************************
+* THE RETURN ADDRESSES ON THIS LETTER HAVE BEEN SET TO PREVENT MAIL *
+* LOOPS IN THE EVENT YOU ARE RUNNING SOFTWARE WHICH AUTO-REPLIES TO *
+* INBOUND MAIL. WE WILL NOT SEE ANY REPLY SENT TO THIS LETTER *
+*********************************************************************
+
+Dear customer,
+
+Please find below the statistics of your traffic consumption for $DATE
+
+---------------------------------------------------------------------
+EOF
+
+ grep 213.130.3.56 $FILE
+
+ cat - <<EOF
+---------------------------------------------------------------------
+
+Kind regards,
+Trafsum,
+Statistics robot
+EOF
+ ) | sendmail some.addr@some.domain
+
diff --git a/trafsum/translate b/trafsum/translate
new file mode 100644
index 0000000..a163d1d
--- /dev/null
+++ b/trafsum/translate
@@ -0,0 +1,7 @@
+# This file is used to translate IP addresses for those customers who
+# have more than one IP assigned
+# Syntax is:
+# <from-ip-addr>:<to-ip-addr>
+# In the example below all traffic for 213.130.4.43 will be summed up
+# with that for 213.130.4.40:
+#213.130.4.43:213.130.4.40 Comment

Return to:

Send suggestions and report system problems to the System administrator.