diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2014-10-18 23:28:25 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2014-10-18 23:28:25 +0300 |
commit | 2166295b26a7b69481148e55b36fedfea84ea413 (patch) | |
tree | 94989b08cc8d9b81b4ba509f69f2959af035b6cc /whoseip | |
parent | d8477a3b0687660b6a90bc97dd86313d2aedf519 (diff) | |
download | dnstools-2166295b26a7b69481148e55b36fedfea84ea413.tar.gz dnstools-2166295b26a7b69481148e55b36fedfea84ea413.tar.bz2 |
whoseip: New options --import and --export
* Makefile.PL: Require Data:Dumper
* Whoseip/DB.pm (ipdb_export,ipdb_import): New functions
(ipdb_insert_unlocked): Last argument is a hash reference.
(ipdb_import,ipdb_export): New functions.
* whoseip.pl: New options --import and --export
Diffstat (limited to 'whoseip')
-rw-r--r-- | whoseip/Makefile.PL | 1 | ||||
-rw-r--r-- | whoseip/Whoseip/DB.pm | 178 | ||||
-rw-r--r-- | whoseip/whoseip.pl | 68 |
3 files changed, 211 insertions, 36 deletions
diff --git a/whoseip/Makefile.PL b/whoseip/Makefile.PL index 395e359..9fb3d76 100644 --- a/whoseip/Makefile.PL +++ b/whoseip/Makefile.PL @@ -23,6 +23,7 @@ use ExtUtils::AutoInstall ( 'Pod::Man' => 2.25, 'Net::CIDR' => 0.14, 'Data::UUID' => 1.219, + 'Data::Dumper' => 2.135_06, 'Storable' => 2.34 ] ); diff --git a/whoseip/Whoseip/DB.pm b/whoseip/Whoseip/DB.pm index 55e86d2..5cc36c9 100644 --- a/whoseip/Whoseip/DB.pm +++ b/whoseip/Whoseip/DB.pm @@ -21,21 +21,38 @@ use Fcntl qw(SEEK_SET SEEK_CUR :flock); use Socket qw(inet_ntoa); use Storable qw(freeze thaw); use Data::UUID; +use Data::Dumper; use Carp; require Exporter; our @ISA = qw(Exporter); our %EXPORT_TAGS = ( 'all' => [ qw(ipdb_open ipdb_lookup ipdb_insert - ipdb_sync ipdb_locker ipdb_close) ] ); + ipdb_sync ipdb_locker ipdb_close + ipdb_export ipdb_import) ] ); our @EXPORT_OK = ( qw(ipdb_open ipdb_lookup ipdb_insert - ipdb_sync ipdb_locker ipdb_close) ); + ipdb_sync ipdb_locker ipdb_close + ipdb_export ipdb_import) ); our @EXPORT = qw(); our $VERSION = "0.1"; +my %ipv4_mask2len; +my @ipv4_len2mask; + +BEGIN { + my $ip = 0xffffffff; + my $masklen = 32; + for (my $i = 32; $i >= 0; $i--) { + $ipv4_mask2len{$ip} = $masklen; + unshift @ipv4_len2mask, $ip; + $ip = ($ip << 1) & 0xffffffff; + --$masklen; + } +} + =pod =head1 NAME @@ -292,6 +309,8 @@ to force syncronization with the disk state. When set to B<0>, disables synchronization with the disk file. Default is B<1>. +=back + =cut sub ipdb_locker { my ($dbf) = shift; @@ -589,13 +608,13 @@ sub ipdb_get_page($$) { $ret{type} = unpack('L', $s); $ret{off} = $off; if ($ret{type} == IPDB_PAGE_INDEX) { - print STDERR "found index page at $off\n" if $dbf->{debug} > 3; + print STDERR "loaded index page at $off\n" if $dbf->{debug} > 3; my ($x, @a) = unpack('LL257', $s); $ret{tab} = \@a; } elsif ($ret{type} == IPDB_PAGE_LEAF) { (my $x, my $nent, $ret{next}) = unpack('LLL', $s); - print STDERR "found leaf page at $off, has $nent entries\n" + print STDERR "loaded leaf page at $off, has $nent entries\n" if $dbf->{debug} > 3; my ($x1, $x2, $x3, @a) = unpack("LLL(LLLa2L/a)$nent", $s); for (my $i = 0; $i < $nent; $i++) { @@ -705,28 +724,42 @@ sub ipdb_lookup_unlocked($$) { if $dbf->{debug}; my $page = ipdb_get_root_page($dbf, $nbits); my $n = 0; - while (1) { - if ($page->{type} == IPDB_PAGE_INDEX) { - print STDERR "index page $page->{off}: ". - join(',', @{$page->{tab}})."\n" - if $dbf->{debug} > 1; - print STDERR "ipdb_lookup: octet ${n}=$ipo[$n], off=$page->{tab}[$ipo[$n]]\n" - if $dbf->{debug}; - if ($page->{tab}[$ipo[$n]]) { - $page = ipdb_cache_get($dbf, $page->{tab}[$ipo[$n]]); - ++$n; - next; - } - return undef if (!$page->{tab}[LEAF_IDX]); - $page = ipdb_cache_get($dbf, $page->{tab}[LEAF_IDX]); + my @leafstk; + while ($page->{type} == IPDB_PAGE_INDEX) { + print STDERR "index page $page->{off}: ". + join(',', @{$page->{tab}})."\n" + if $dbf->{debug} > 1; + print STDERR "ipdb_lookup: octet ${n}=$ipo[$n], off=$page->{tab}[$ipo[$n]]\n" + if $dbf->{debug}; + push @leafstk, $page->{tab}[LEAF_IDX] if $page->{tab}[LEAF_IDX]; + if ($page->{tab}[$ipo[$n]]) { + $page = ipdb_cache_get($dbf, $page->{tab}[$ipo[$n]]); + ++$n; + } else { + last; } + } + + if ($dbf->{debug}) { + print STDERR "ipdb_lookup: $ipstr: descended to ". + join('.', @ipo[0..$n]); + if ($#leafstk >= 0) { + print STDERR ", now rescanning leaf stack (".($#leafstk + 1). + " entries)\n"; + } else { + print STDERR ", found nothing"; + } + print STDERR "\n"; + } + + while (my $off = pop @leafstk) { + $page = ipdb_cache_get($dbf, $off); my $i = 0; foreach my $r (@{$page->{tab}}) { print STDERR "ipdb_lookup: compare ($ipn & $r->[1]) == $r->[0]\n" if $dbf->{debug}; if (($ipn & $r->[1]) == $r->[0]) { - # FIXME: check timestamp print STDERR "ipdb_lookup: MATCH $r->[3]\n" if $dbf->{debug}; if (defined($dbf->{ttl}) and @@ -746,9 +779,9 @@ sub ipdb_lookup_unlocked($$) { } ++$i; } - return undef if (!$page->{next}); - $page = ipdb_cache_get($dbf, $page->{next}); + push @leafstk, $page->{next} if ($page->{next}); } + return undef; } sub ipdb_lookup($$) { @@ -769,18 +802,16 @@ sub ipdb_lookup($$) { =pod -=head2 $res = B<Whoseip::DB::ipdb_insert(I<$dbf>, I<$cidr>, I<$country>);> +=head2 $res = B<Whoseip::DB::ipdb_insert(I<$dbf>, I<$cidr>, I<$country>>[B<, I<$hashref>>]B<);> -Inserts into the database I<$cidr> and the corresponding country code I<$country>. +Inserts into the database I<$cidr> and the corresponding country code I<$country> +and additional data I<$hashref>. Currently, I<$cidr> must be in the form B<I<Net-address>/I<Netmask-length>>. =cut sub ipdb_insert_unlocked { - my $dbf = shift; - my $cidr = shift; - my $country = shift; - local %_ = @_; + my ($dbf, $cidr, $country, $href) = @_; my @ipo; my $ipn; my $masklen; @@ -800,7 +831,7 @@ sub ipdb_insert_unlocked { return 0; } - print STDERR "inserting $cidr $country\n" if $dbf->{debug}; + print STDERR "ipdb_insert: inserting $cidr $country\n" if $dbf->{debug}; my $n = int($masklen / 8); @@ -833,7 +864,7 @@ sub ipdb_insert_unlocked { $page = $p; } - push @{$page->{tab}}, [ $ipn, $netmask, time(), $country, \%_ ]; + push @{$page->{tab}}, [ $ipn, $netmask, time(), $country, $href ]; $page->{dirty} = 1; return 1; @@ -851,5 +882,94 @@ sub ipdb_insert { return $res; } +sub ipdb_dump_page { + my ($dbf, $off, $fd) = @_; + my $page = ipdb_get_page($dbf, $off); + if ($page->{type} == IPDB_PAGE_INDEX) { + ipdb_dump_page($dbf, $page->{tab}[LEAF_IDX], $fd) + if $page->{tab}[LEAF_IDX]; + foreach my $off (@{$page->{tab}}[0 .. LEAF_IDX - 1]) { + ipdb_dump_page($dbf, $off, $fd) if $off; + } + } elsif ($page->{type} == IPDB_PAGE_LEAF) { + do { + foreach my $r (@{$page->{tab}}) { + $r->[0] = inet_ntoa(pack('N', $r->[0])); + if (defined($ipv4_mask2len{$r->[1]})) { + $r->[1] = $ipv4_mask2len{$r->[1]}; + my $v = Dumper($r); + print $fd "$v\n"; + } else { + print STDERR "ignoring invalid entry " . + $r->[0] . '/' . + inet_ntoa(pack('N', $r->[1])) + } + } + $page = $page->{next} ? ipdb_get_page($dbf, $page->{next}) : undef; + } while ($page); + } +} + +=pod + +=head1 B<Whoseip::DB::ipdb_export(I<$dbf>>[B<, I<$fd>>]B<);> + +Exports database I<$dbf> in a portable plain-text format to +file identified by I<$fd> (B<STDOUT> by default). + +The created file can be transferred over the network and used +to recreate the database via B<Whoseip::DB::ipdb_import>. + +=cut + +sub ipdb_export { + my ($dbf, $fd) = @_; + + $fd = *STDOUT unless defined($fd); + + my $ug = new Data::UUID; + + print $fd "# Whoseip dump from file $dbf->{filename}\n"; + print $fd "# UUID ".$ug->to_string($dbf->{uuid})."\n"; + local $Data::Dumper::Indent = 0; + local $Data::Dumper::Terse = 1; + foreach my $off (values %{$dbf->{rootidx}}) { + ipdb_dump_page($dbf, $off, $fd); + } +} + +=pod + +=head1 B<$res = Whoseip::DB::ipdb_import(I<$dbf>>[B<, I<$fd>>]B<);> + +Imports data from the file descriptor I<$fd> (B<STDIN> by default) +into the database file I<$dbf>. Returns 1 on success and 0 on +failure. + +=cut + +sub ipdb_import { + my ($dbf, $fd) = @_; + + $fd = *STDIN unless defined($fd); + + ipdb_locker($dbf, lock => LOCK_EX); + my $line = 0; + while (<$fd>) { + ++$line; + chomp; + next if /^#/; + my $r = eval $_; + if ($@) { + print STDERR "ipdb_import: error in line $line: $@\n"; + return 0; + } + my $cidr = "$r->[0]/$r->[1]"; + ipdb_insert_unlocked($dbf, $cidr, $r->[3], $r->[4]); + } + ipdb_sync($dbf); + ipdb_locker($dbf, lock => LOCK_UN); + return 1; +} 1; diff --git a/whoseip/whoseip.pl b/whoseip/whoseip.pl index 78f7f8a..aa55770 100644 --- a/whoseip/whoseip.pl +++ b/whoseip/whoseip.pl @@ -43,6 +43,7 @@ my $delim = $LF; # Output delimiter my $dbf; my $dbfile; +my %dbopt; my %fmtab = (unix => '${status} $?{diag}{${diag}}{${country} ${cidr} ${range} ${count}} ', @@ -499,10 +500,10 @@ sub serve { $res{status} = 'OK'; if (defined($dbf)) { foreach my $cidr (split /,/, $res{cidr}) { - ipdb_insert($dbf, $cidr, $res{country}, - cidr => $res{cidr}, - server => $res{server}, - port => $res{port}); + ipdb_insert($dbf, $cidr, uc $res{country}, + { cidr => $res{cidr}, + server => $res{server}, + port => $res{port} }); } } } @@ -683,7 +684,8 @@ sub docgi { my $output_format; my $fastcgi; my $single_query; -my $cache_ttl; +my $dbexport; +my $dbimport; if (defined($ENV{WHOSEIP_CONF})) { read_config_file($ENV{WHOSEIP_CONF}); @@ -721,12 +723,46 @@ GetOptions("h" => sub { "cache-file|c:s" => \$dbfile, "no-cache|N" => sub { $dbfile = undef; }, "single-query" => \$single_query, - "cache-ttl|ttl|t=n" => \$cache_ttl, + "cache-ttl|ttl|t=n" => sub { + $dbopt{ttl} => $_[1]; + }, + "cache-mode=s" => sub { + $dbopt{mode} = $_[1]; + }, + "export" => \$dbexport, + "import" => \$dbimport, ) or exit(EX_USAGE); if (defined($dbfile)) { $dbfile .= "whoseip.db" if (-d $dbfile); - $dbf = ipdb_open($dbfile, debug => $debug, ttl => $cache_ttl); + $dbopt{debug} = $debug; + $dbf = ipdb_open($dbfile, %dbopt); +} + +if (defined($dbexport)) { + abend(EX_USAGE, "--export requires --cache-file") unless defined($dbf); + abend(EX_USAGE, "too many arguments") if ($#ARGV > 0); + my $fd; + if ($#ARGV == 0) { + open($fd, '>', $ARGV[0]) or + abend(EX_CANTCREAT, "can't open $ARGV[0] for writing: $!"); + } + ipdb_export($dbf, $fd); + ipdb_close($dbf); + exit(EX_OK); +} + +if (defined($dbimport)) { + abend(EX_USAGE, "--import requires --cache-file") unless defined($dbf); + abend(EX_USAGE, "too many arguments") if ($#ARGV > 0); + my $fd; + if ($#ARGV == 0) { + open($fd, '<', $ARGV[0]) or + abend(EX_NOINPUT, "can't open $ARGV[0] for reading: $!"); + } + ipdb_import($dbf, $fd); + ipdb_close($dbf); + exit(EX_OK); } if (defined($fastcgi)) { @@ -815,11 +851,13 @@ B<whoiseip> [B<--debug>] [B<--define-format=>I<NAME>B<=>I<TEXT>] [B<--dump=>I<FILE>] +[B<--export>] [B<--fastcgi=>[I<SUFFIX...>]] [B<--format=>I<TEXT>] [B<--format-file=>[I<NAME>B<=>]I<FILE>] [B<--formfile=>I<FILE>] [B<--help>] +[B<--import>] [B<--ip-list=>I<FILE>] [B<--no-cache>] [B<--single-query>] @@ -970,6 +1008,16 @@ format B<cgi> is used to respond to B<CGI> or B<FastCGI> requests, and format B<unix> is used when serving requests coming from command line or in inetd mode. See the section B<FORMAT>, for a detailed discussion. +=item B<--export> + +Export the IP database into portable ASCII dump file. If a singe argument +is supplied, it gives the name of the output file. In the absense of +arguments, the output goes to the standard output. + +The created file can be transmitted over the network to hosts of another +architecture and used there to recreate the database, using the +B<whoseip --import>. + =item B<--fastcgi=>[I<SUFFIX...>] When used without argument, forces FastCGI mode. If an argument is given, @@ -1004,6 +1052,12 @@ Comments are introduced with a B<#> sign. Empty lines are ignored. Without this option, B<whoseip> uses the built-in list of servers. +=item B<--import> + +Import data from the file given as the first argument into the database. If +no argument is given, read from standard input. The input must be a valid +B<whoseip> database dump, as produced by B<whoseip --export>. + =item B<-N>, B<--no-cache> Disable caching (this is the default). |