diff options
-rw-r--r-- | trafsum/README | 20 | ||||
-rw-r--r-- | trafsum/traflog.awk | 45 | ||||
-rwxr-xr-x | trafsum/trafsum | 160 | ||||
-rw-r--r-- | trafsum/trafsum.conf | 11 | ||||
-rw-r--r-- | trafsum/trafsum.local | 50 | ||||
-rw-r--r-- | trafsum/translate | 7 |
6 files changed, 293 insertions, 0 deletions
diff --git a/trafsum/README b/trafsum/README new file mode 100644 index 0000000..cf7f56a --- /dev/null +++ b/trafsum/README | |||
@@ -0,0 +1,20 @@ | |||
1 | This is trafsum: a traffic data summator. It processes the information | ||
2 | from one or several log files and stores the data in MySQL database. The | ||
3 | format of log files is as follows: | ||
4 | |||
5 | # <---- Slice time ----> IP-Address Inbytes Outbytes | ||
6 | Mon Jun 4 03:30:02 2001 213.130.7.7 4012128 7627102 | ||
7 | |||
8 | To install: | ||
9 | |||
10 | 1. Copy the entire directory somewhere where you'd like it to be installed; | ||
11 | 2. Copy the file trafsum.conf to /etc and edit it to your liking. It | ||
12 | should have access permissions of 600. | ||
13 | 3. If necessary, you may wish to edit <instdir>/translate. The necessary | ||
14 | instructons are found in the comment at the start of the file; | ||
15 | 4. Add the script to your crontab: | ||
16 | |||
17 | 20 0 * * * /usr/local/trafsum/trafsum >/var/log/trafsum.err 2>&1 | ||
18 | |||
19 | 5. Take cover and wait for the script to work. | ||
20 | |||
diff --git a/trafsum/traflog.awk b/trafsum/traflog.awk new file mode 100644 index 0000000..7792fb9 --- /dev/null +++ b/trafsum/traflog.awk | |||
@@ -0,0 +1,45 @@ | |||
1 | # $Id: traflog.awk,v 1.1 2005/08/04 07:15:38 gray Exp $ | ||
2 | # Copyright 2000, 2005 Sergey Poznyakoff | ||
3 | # | ||
4 | # This program is free software; you can redistribute it and/or modify | ||
5 | # it under the terms of the GNU General Public License as published by | ||
6 | # the Free Software Foundation; either version 2, or (at your option) | ||
7 | # any later version. | ||
8 | # | ||
9 | # This program is distributed in the hope that it will be useful, | ||
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | # GNU General Public License for more details. | ||
13 | # | ||
14 | # You should have received a copy of the GNU General Public License | ||
15 | # along with this program; if not, write to the Free Software | ||
16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, | ||
17 | # MA 02110-1301, USA. | ||
18 | |||
19 | #Field assignment | ||
20 | # 1 2 3 4 5 6 7 8 | ||
21 | #Sun Jun 20 00:04:06 1999 208.244.48.32 30 1736 | ||
22 | |||
23 | BEGIN { | ||
24 | divisor = 1048576 | ||
25 | maxtraf = 4294960000 | ||
26 | } | ||
27 | |||
28 | NF==8 { | ||
29 | # Sanity check | ||
30 | if (0+$7 < 0 || 0+$8 < 0 || 0+$7 > maxtraf || 0+$8 > maxtraf) { | ||
31 | print NR ": Skipped (" $7 " " $8 ")" >"/dev/stderr" | ||
32 | next | ||
33 | } else { | ||
34 | inbytes[$6] += 0+$7 | ||
35 | outbytes[$6] += 0+$8 | ||
36 | } | ||
37 | } | ||
38 | |||
39 | END { | ||
40 | for (ip in inbytes) { | ||
41 | print ip " " inbytes[ip] / divisor " " outbytes[ip] / divisor | ||
42 | } | ||
43 | } | ||
44 | |||
45 | |||
diff --git a/trafsum/trafsum b/trafsum/trafsum new file mode 100755 index 0000000..10437bd --- /dev/null +++ b/trafsum/trafsum | |||
@@ -0,0 +1,160 @@ | |||
1 | #! /bin/sh | ||
2 | # Copyright 2000, 2005 Sergey Poznyakoff | ||
3 | # | ||
4 | # This program is free software; you can redistribute it and/or modify | ||
5 | # it under the terms of the GNU General Public License as published by | ||
6 | # the Free Software Foundation; either version 2, or (at your option) | ||
7 | # any later version. | ||
8 | # | ||
9 | # This program is distributed in the hope that it will be useful, | ||
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | # GNU General Public License for more details. | ||
13 | # | ||
14 | # You should have received a copy of the GNU General Public License | ||
15 | # along with this program; if not, write to the Free Software | ||
16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, | ||
17 | # MA 02110-1301, USA. | ||
18 | |||
19 | PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin | ||
20 | export PATH | ||
21 | |||
22 | BASEDIR=`dirname $0` | ||
23 | SED=/tmp/trafsum.$$.sed | ||
24 | TEMP=/tmp/trafsum.$$ | ||
25 | TERR=/tmp/trafsum.err.$$ | ||
26 | CONF=/etc/trafsum.conf | ||
27 | SUMMARY=/tmp/trafsum.summary | ||
28 | LOG=/var/log/ipstat/trafsum.log | ||
29 | SQLLOG=/var/log/ipstat/trafsum.sql | ||
30 | DATE=`date +%y%m%d%n -d yesterday` | ||
31 | STDERR=/var/log/ipstat/trafsum.err | ||
32 | HISTORY=/var/log/ipstat/history | ||
33 | |||
34 | log() { | ||
35 | echo "$*" | tee -a $TEMPLOG >> $LOG | ||
36 | } | ||
37 | |||
38 | ## Determine yesterday's date | ||
39 | |||
40 | yesterday() { | ||
41 | datestr=`date +%Y-%m-%d -d "-1 day"` | ||
42 | } | ||
43 | |||
44 | lookup() { | ||
45 | NAME=`nslookup $1 2>/dev/null | sed -ne 's/Name: *\(.*\)/\1/p'` | ||
46 | if [ "$NAME" != "" ]; then | ||
47 | cp $BASEDIR/namelist $BASEDIR/namelist.b | ||
48 | if fgrep "$1:" $BASEDIR/namelist 2>/dev/null ; then | ||
49 | sed -e "s/$1:.*$/$1: $NAME/" $BASEDIR/namelist.b > $BASEDIR/namelist | ||
50 | else | ||
51 | echo "$1: $NAME" >> $BASEDIR/namelist | ||
52 | fi | ||
53 | log "Cached DNS entry $1: $NAME" | ||
54 | else | ||
55 | NAME=$1 | ||
56 | fi | ||
57 | } | ||
58 | |||
59 | ########################################### | ||
60 | ### Main | ||
61 | # | ||
62 | |||
63 | . $CONF | ||
64 | |||
65 | trap 'rm -f $SED $TEMP $TEMPLOG $SUMMARY' 0 1 2 3 15 | ||
66 | |||
67 | if [ x"$SOURCE_FILES" = x"" ]; then | ||
68 | log "Configuration error: no source files specified" | ||
69 | exit 1 | ||
70 | fi | ||
71 | cat /dev/null > $SUMMARY | ||
72 | cat /dev/null > $STDERR | ||
73 | |||
74 | if [ ! -f $LOG ] ; then | ||
75 | cat /dev/null > $LOG | ||
76 | fi | ||
77 | |||
78 | cat $SOURCE_FILES > $SUMMARY | ||
79 | |||
80 | # Get yesterday's date into $datestr | ||
81 | yesterday | ||
82 | |||
83 | echo -e "\ndate: $datestr" | ||
84 | |||
85 | ## Process statistics data | ||
86 | # | ||
87 | if [ -r $BASEDIR/translate ]; then | ||
88 | sed -ne 's%\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\):[^0-9]*\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\).*%s/\1\\.\2\\.\3\\.\4/\5\\.\6\\.\7\\.\8/%p' $BASEDIR/translate > $SED | ||
89 | else | ||
90 | rm -f $SED # Make sure it does not exist | ||
91 | fi | ||
92 | |||
93 | MYSQL_OPTIONS="-A " | ||
94 | if [ "$SERVER" ]; then | ||
95 | MYSQL_OPTIONS="$MYSQL_OPTIONS -h$SERVER" | ||
96 | fi | ||
97 | |||
98 | if [ "$PORT" ]; then | ||
99 | MYSQL_OPTIONS="$MYSQL_OPTIONS -P$PORT" | ||
100 | fi | ||
101 | |||
102 | if [ "$MYSQL_USER" ]; then | ||
103 | MYSQL_OPTIONS="$MYSQL_OPTIONS -u$MYSQL_USER" | ||
104 | fi | ||
105 | |||
106 | if [ "$MYSQL_PASSWORD" ]; then | ||
107 | MYSQL_OPTIONS="$MYSQL_OPTIONS -p$MYSQL_PASSWORD" | ||
108 | fi | ||
109 | MYSQL_OPTIONS="$MYSQL_OPTIONS $DATABASE" | ||
110 | |||
111 | cat $SUMMARY | | ||
112 | ( | ||
113 | if [ -r $SED ]; then | ||
114 | sed -f $SED | ||
115 | else | ||
116 | cat - | ||
117 | fi | \ | ||
118 | awk -f $BASEDIR/traflog.awk 2>$TERR | | ||
119 | while read IPADDR INBYTES OUTBYTES | ||
120 | do | ||
121 | NAME=`fgrep "$IPADDR:" $BASEDIR/namelist | sed -e 's/.*: *\(.*\)/\1/' 2>/dev/null` | ||
122 | [ "$NAME" = "" ] && lookup $IPADDR | ||
123 | echo -e "$IPADDR\t$NAME\t$INBYTES\t\t$OUTBYTES" >> $TEMP | ||
124 | echo "INSERT INTO $TABLE VALUES ('$IPADDR','$datestr',$INBYTES,$OUTBYTES);" | ||
125 | done | ||
126 | ) | | ||
127 | tee $SQLLOG | | ||
128 | mysql $MYSQL_OPTIONS | ||
129 | |||
130 | [ x"$ADDRESSLIST" != x"" ] && \ | ||
131 | ( | ||
132 | if [ -s $TERR ]; then | ||
133 | echo "WARNING: traflog.awk reported errors. They are:" | ||
134 | echo "===============================================" | ||
135 | cat $TERR | ||
136 | echo "===============================================" | ||
137 | fi | ||
138 | |||
139 | sort -t'.' -n +0 -1 +1 -2 +2 -3 +3 -4 $TEMP | \ | ||
140 | awk -v DATE=$datestr 'BEGIN { | ||
141 | print "I/O traffic summary for " DATE | ||
142 | |||
143 | printf "%-17.17s%24.24s%15s%15s\n", "IP", "Name/address", "Incoming", "Outgoing" | ||
144 | } | ||
145 | { printf "%-17.17s%24.24s%15s%15s\n", $1, $2, $3, $4 }' \ | ||
146 | ) | mail -s "traffic statistics" $ADDRESSLIST | ||
147 | |||
148 | cat $SUMMARY | gzip - > $HISTORY/${DATE}.gz | ||
149 | |||
150 | if [ -x $BASEDIR/trafsum.local ]; then | ||
151 | sh $BASEDIR/trafsum.local $SUMMARY | ||
152 | fi | ||
153 | |||
154 | rm -f $SED $TEMP $TEMPLOG $SUMMARY $TERR | ||
155 | for i in $SOURCE_FILES | ||
156 | do | ||
157 | cat /dev/null > $i | ||
158 | done | ||
159 | |||
160 | |||
diff --git a/trafsum/trafsum.conf b/trafsum/trafsum.conf new file mode 100644 index 0000000..dd1df10 --- /dev/null +++ b/trafsum/trafsum.conf | |||
@@ -0,0 +1,11 @@ | |||
1 | # Config file for traffic accounter | ||
2 | |||
3 | SERVER=#<MYSQL server name> | ||
4 | PORT=#<MYSQL port> | ||
5 | DATABASE=#<Database name> | ||
6 | TABLE=#<Table name> | ||
7 | MYSQL_USER= | ||
8 | MYSQL_PASSWORD= | ||