aboutsummaryrefslogtreecommitdiff
path: root/whoseip
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org.ua>2014-10-18 23:28:25 +0300
committerSergey Poznyakoff <gray@gnu.org.ua>2014-10-18 23:28:25 +0300
commit2166295b26a7b69481148e55b36fedfea84ea413 (patch)
tree94989b08cc8d9b81b4ba509f69f2959af035b6cc /whoseip
parentd8477a3b0687660b6a90bc97dd86313d2aedf519 (diff)
downloaddnstools-2166295b26a7b69481148e55b36fedfea84ea413.tar.gz
dnstools-2166295b26a7b69481148e55b36fedfea84ea413.tar.bz2
whoseip: New options --import and --export
* Makefile.PL: Require Data:Dumper * Whoseip/DB.pm (ipdb_export,ipdb_import): New functions (ipdb_insert_unlocked): Last argument is a hash reference. (ipdb_import,ipdb_export): New functions. * whoseip.pl: New options --import and --export
Diffstat (limited to 'whoseip')
-rw-r--r--whoseip/Makefile.PL1
-rw-r--r--whoseip/Whoseip/DB.pm178
-rw-r--r--whoseip/whoseip.pl68
3 files changed, 211 insertions, 36 deletions
diff --git a/whoseip/Makefile.PL b/whoseip/Makefile.PL
index 395e359..9fb3d76 100644
--- a/whoseip/Makefile.PL
+++ b/whoseip/Makefile.PL
@@ -23,6 +23,7 @@ use ExtUtils::AutoInstall (
'Pod::Man' => 2.25,
'Net::CIDR' => 0.14,
'Data::UUID' => 1.219,
+ 'Data::Dumper' => 2.135_06,
'Storable' => 2.34
]
);
diff --git a/whoseip/Whoseip/DB.pm b/whoseip/Whoseip/DB.pm
index 55e86d2..5cc36c9 100644
--- a/whoseip/Whoseip/DB.pm
+++ b/whoseip/Whoseip/DB.pm
@@ -21,21 +21,38 @@ use Fcntl qw(SEEK_SET SEEK_CUR :flock);
use Socket qw(inet_ntoa);
use Storable qw(freeze thaw);
use Data::UUID;
+use Data::Dumper;
use Carp;
require Exporter;
our @ISA = qw(Exporter);
our %EXPORT_TAGS = ( 'all' => [ qw(ipdb_open ipdb_lookup ipdb_insert
- ipdb_sync ipdb_locker ipdb_close) ] );
+ ipdb_sync ipdb_locker ipdb_close
+ ipdb_export ipdb_import) ] );
our @EXPORT_OK = ( qw(ipdb_open ipdb_lookup ipdb_insert
- ipdb_sync ipdb_locker ipdb_close) );
+ ipdb_sync ipdb_locker ipdb_close
+ ipdb_export ipdb_import) );
our @EXPORT = qw();
our $VERSION = "0.1";
+my %ipv4_mask2len;
+my @ipv4_len2mask;
+
+BEGIN {
+ my $ip = 0xffffffff;
+ my $masklen = 32;
+ for (my $i = 32; $i >= 0; $i--) {
+ $ipv4_mask2len{$ip} = $masklen;
+ unshift @ipv4_len2mask, $ip;
+ $ip = ($ip << 1) & 0xffffffff;
+ --$masklen;
+ }
+}
+
=pod
=head1 NAME
@@ -292,6 +309,8 @@ to force syncronization with the disk state.
When set to B<0>, disables synchronization with the disk file. Default is
B<1>.
+=back
+
=cut
sub ipdb_locker {
my ($dbf) = shift;
@@ -589,13 +608,13 @@ sub ipdb_get_page($$) {
$ret{type} = unpack('L', $s);
$ret{off} = $off;
if ($ret{type} == IPDB_PAGE_INDEX) {
- print STDERR "found index page at $off\n" if $dbf->{debug} > 3;
+ print STDERR "loaded index page at $off\n" if $dbf->{debug} > 3;
my ($x, @a) = unpack('LL257', $s);
$ret{tab} = \@a;
} elsif ($ret{type} == IPDB_PAGE_LEAF) {
(my $x, my $nent, $ret{next}) =
unpack('LLL', $s);
- print STDERR "found leaf page at $off, has $nent entries\n"
+ print STDERR "loaded leaf page at $off, has $nent entries\n"
if $dbf->{debug} > 3;
my ($x1, $x2, $x3, @a) = unpack("LLL(LLLa2L/a)$nent", $s);
for (my $i = 0; $i < $nent; $i++) {
@@ -705,28 +724,42 @@ sub ipdb_lookup_unlocked($$) {
if $dbf->{debug};
my $page = ipdb_get_root_page($dbf, $nbits);
my $n = 0;
- while (1) {
- if ($page->{type} == IPDB_PAGE_INDEX) {
- print STDERR "index page $page->{off}: ".
- join(',', @{$page->{tab}})."\n"
- if $dbf->{debug} > 1;
- print STDERR "ipdb_lookup: octet ${n}=$ipo[$n], off=$page->{tab}[$ipo[$n]]\n"
- if $dbf->{debug};
- if ($page->{tab}[$ipo[$n]]) {
- $page = ipdb_cache_get($dbf, $page->{tab}[$ipo[$n]]);
- ++$n;
- next;
- }
- return undef if (!$page->{tab}[LEAF_IDX]);
- $page = ipdb_cache_get($dbf, $page->{tab}[LEAF_IDX]);
+ my @leafstk;
+ while ($page->{type} == IPDB_PAGE_INDEX) {
+ print STDERR "index page $page->{off}: ".
+ join(',', @{$page->{tab}})."\n"
+ if $dbf->{debug} > 1;
+ print STDERR "ipdb_lookup: octet ${n}=$ipo[$n], off=$page->{tab}[$ipo[$n]]\n"
+ if $dbf->{debug};
+ push @leafstk, $page->{tab}[LEAF_IDX] if $page->{tab}[LEAF_IDX];
+ if ($page->{tab}[$ipo[$n]]) {
+ $page = ipdb_cache_get($dbf, $page->{tab}[$ipo[$n]]);
+ ++$n;
+ } else {
+ last;
}
+ }
+
+ if ($dbf->{debug}) {
+ print STDERR "ipdb_lookup: $ipstr: descended to ".
+ join('.', @ipo[0..$n]);
+ if ($#leafstk >= 0) {
+ print STDERR ", now rescanning leaf stack (".($#leafstk + 1).
+ " entries)\n";
+ } else {
+ print STDERR ", found nothing";
+ }
+ print STDERR "\n";
+ }
+
+ while (my $off = pop @leafstk) {
+ $page = ipdb_cache_get($dbf, $off);
my $i = 0;
foreach my $r (@{$page->{tab}}) {
print STDERR "ipdb_lookup: compare ($ipn & $r->[1]) == $r->[0]\n"
if $dbf->{debug};
if (($ipn & $r->[1]) == $r->[0]) {
- # FIXME: check timestamp
print STDERR "ipdb_lookup: MATCH $r->[3]\n"
if $dbf->{debug};
if (defined($dbf->{ttl}) and
@@ -746,9 +779,9 @@ sub ipdb_lookup_unlocked($$) {
}
++$i;
}
- return undef if (!$page->{next});
- $page = ipdb_cache_get($dbf, $page->{next});
+ push @leafstk, $page->{next} if ($page->{next});
}
+ return undef;
}
sub ipdb_lookup($$) {
@@ -769,18 +802,16 @@ sub ipdb_lookup($$) {
=pod
-=head2 $res = B<Whoseip::DB::ipdb_insert(I<$dbf>, I<$cidr>, I<$country>);>
+=head2 $res = B<Whoseip::DB::ipdb_insert(I<$dbf>, I<$cidr>, I<$country>>[B<, I<$hashref>>]B<);>
-Inserts into the database I<$cidr> and the corresponding country code I<$country>.
+Inserts into the database I<$cidr> and the corresponding country code I<$country>
+and additional data I<$hashref>.
Currently, I<$cidr> must be in the form B<I<Net-address>/I<Netmask-length>>.
=cut
sub ipdb_insert_unlocked {
- my $dbf = shift;
- my $cidr = shift;
- my $country = shift;
- local %_ = @_;
+ my ($dbf, $cidr, $country, $href) = @_;
my @ipo;
my $ipn;
my $masklen;
@@ -800,7 +831,7 @@ sub ipdb_insert_unlocked {
return 0;
}
- print STDERR "inserting $cidr $country\n" if $dbf->{debug};
+ print STDERR "ipdb_insert: inserting $cidr $country\n" if $dbf->{debug};
my $n = int($masklen / 8);
@@ -833,7 +864,7 @@ sub ipdb_insert_unlocked {
$page = $p;
}
- push @{$page->{tab}}, [ $ipn, $netmask, time(), $country, \%_ ];
+ push @{$page->{tab}}, [ $ipn, $netmask, time(), $country, $href ];
$page->{dirty} = 1;
return 1;
@@ -851,5 +882,94 @@ sub ipdb_insert {
return $res;
}
+sub ipdb_dump_page {
+ my ($dbf, $off, $fd) = @_;
+ my $page = ipdb_get_page($dbf, $off);
+ if ($page->{type} == IPDB_PAGE_INDEX) {
+ ipdb_dump_page($dbf, $page->{tab}[LEAF_IDX], $fd)
+ if $page->{tab}[LEAF_IDX];
+ foreach my $off (@{$page->{tab}}[0 .. LEAF_IDX - 1]) {
+ ipdb_dump_page($dbf, $off, $fd) if $off;
+ }
+ } elsif ($page->{type} == IPDB_PAGE_LEAF) {
+ do {
+ foreach my $r (@{$page->{tab}}) {
+ $r->[0] = inet_ntoa(pack('N', $r->[0]));
+ if (defined($ipv4_mask2len{$r->[1]})) {
+ $r->[1] = $ipv4_mask2len{$r->[1]};
+ my $v = Dumper($r);
+ print $fd "$v\n";
+ } else {
+ print STDERR "ignoring invalid entry " .
+ $r->[0] . '/' .
+ inet_ntoa(pack('N', $r->[1]))
+ }
+ }
+ $page = $page->{next} ? ipdb_get_page($dbf, $page->{next}) : undef;
+ } while ($page);
+ }
+}
+
+=pod
+
+=head1 B<Whoseip::DB::ipdb_export(I<$dbf>>[B<, I<$fd>>]B<);>
+
+Exports database I<$dbf> in a portable plain-text format to
+file identified by I<$fd> (B<STDOUT> by default).
+
+The created file can be transferred over the network and used
+to recreate the database via B<Whoseip::DB::ipdb_import>.
+
+=cut
+
+sub ipdb_export {
+ my ($dbf, $fd) = @_;
+
+ $fd = *STDOUT unless defined($fd);
+
+ my $ug = new Data::UUID;
+
+ print $fd "# Whoseip dump from file $dbf->{filename}\n";
+ print $fd "# UUID ".$ug->to_string($dbf->{uuid})."\n";
+ local $Data::Dumper::Indent = 0;
+ local $Data::Dumper::Terse = 1;
+ foreach my $off (values %{$dbf->{rootidx}}) {
+ ipdb_dump_page($dbf, $off, $fd);
+ }
+}
+
+=pod
+
+=head1 B<$res = Whoseip::DB::ipdb_import(I<$dbf>>[B<, I<$fd>>]B<);>
+
+Imports data from the file descriptor I<$fd> (B<STDIN> by default)
+into the database file I<$dbf>. Returns 1 on success and 0 on
+failure.
+
+=cut
+
+sub ipdb_import {
+ my ($dbf, $fd) = @_;
+
+ $fd = *STDIN unless defined($fd);
+
+ ipdb_locker($dbf, lock => LOCK_EX);
+ my $line = 0;
+ while (<$fd>) {
+ ++$line;
+ chomp;
+ next if /^#/;
+ my $r = eval $_;
+ if ($@) {
+ print STDERR "ipdb_import: error in line $line: $@\n";
+ return 0;
+ }
+ my $cidr = "$r->[0]/$r->[1]";
+ ipdb_insert_unlocked($dbf, $cidr, $r->[3], $r->[4]);
+ }
+ ipdb_sync($dbf);
+ ipdb_locker($dbf, lock => LOCK_UN);
+ return 1;
+}
1;
diff --git a/whoseip/whoseip.pl b/whoseip/whoseip.pl
index 78f7f8a..aa55770 100644
--- a/whoseip/whoseip.pl
+++ b/whoseip/whoseip.pl
@@ -43,6 +43,7 @@ my $delim = $LF; # Output delimiter
my $dbf;
my $dbfile;
+my %dbopt;
my %fmtab = (unix => '${status} $?{diag}{${diag}}{${country} ${cidr} ${range} ${count}}
',
@@ -499,10 +500,10 @@ sub serve {
$res{status} = 'OK';
if (defined($dbf)) {
foreach my $cidr (split /,/, $res{cidr}) {
- ipdb_insert($dbf, $cidr, $res{country},
- cidr => $res{cidr},
- server => $res{server},
- port => $res{port});
+ ipdb_insert($dbf, $cidr, uc $res{country},
+ { cidr => $res{cidr},
+ server => $res{server},
+ port => $res{port} });
}
}
}
@@ -683,7 +684,8 @@ sub docgi {
my $output_format;
my $fastcgi;
my $single_query;
-my $cache_ttl;
+my $dbexport;
+my $dbimport;
if (defined($ENV{WHOSEIP_CONF})) {
read_config_file($ENV{WHOSEIP_CONF});
@@ -721,12 +723,46 @@ GetOptions("h" => sub {
"cache-file|c:s" => \$dbfile,
"no-cache|N" => sub { $dbfile = undef; },
"single-query" => \$single_query,
- "cache-ttl|ttl|t=n" => \$cache_ttl,
+ "cache-ttl|ttl|t=n" => sub {
+ $dbopt{ttl} => $_[1];
+ },
+ "cache-mode=s" => sub {
+ $dbopt{mode} = $_[1];
+ },
+ "export" => \$dbexport,
+ "import" => \$dbimport,
) or exit(EX_USAGE);
if (defined($dbfile)) {
$dbfile .= "whoseip.db" if (-d $dbfile);
- $dbf = ipdb_open($dbfile, debug => $debug, ttl => $cache_ttl);
+ $dbopt{debug} = $debug;
+ $dbf = ipdb_open($dbfile, %dbopt);
+}
+
+if (defined($dbexport)) {
+ abend(EX_USAGE, "--export requires --cache-file") unless defined($dbf);
+ abend(EX_USAGE, "too many arguments") if ($#ARGV > 0);
+ my $fd;
+ if ($#ARGV == 0) {
+ open($fd, '>', $ARGV[0]) or
+ abend(EX_CANTCREAT, "can't open $ARGV[0] for writing: $!");
+ }
+ ipdb_export($dbf, $fd);
+ ipdb_close($dbf);
+ exit(EX_OK);
+}
+
+if (defined($dbimport)) {
+ abend(EX_USAGE, "--import requires --cache-file") unless defined($dbf);
+ abend(EX_USAGE, "too many arguments") if ($#ARGV > 0);
+ my $fd;
+ if ($#ARGV == 0) {
+ open($fd, '<', $ARGV[0]) or
+ abend(EX_NOINPUT, "can't open $ARGV[0] for reading: $!");
+ }
+ ipdb_import($dbf, $fd);
+ ipdb_close($dbf);
+ exit(EX_OK);
}
if (defined($fastcgi)) {
@@ -815,11 +851,13 @@ B<whoiseip>
[B<--debug>]
[B<--define-format=>I<NAME>B<=>I<TEXT>]
[B<--dump=>I<FILE>]
+[B<--export>]
[B<--fastcgi=>[I<SUFFIX...>]]
[B<--format=>I<TEXT>]
[B<--format-file=>[I<NAME>B<=>]I<FILE>]
[B<--formfile=>I<FILE>]
[B<--help>]
+[B<--import>]
[B<--ip-list=>I<FILE>]
[B<--no-cache>]
[B<--single-query>]
@@ -970,6 +1008,16 @@ format B<cgi> is used to respond to B<CGI> or B<FastCGI> requests, and
format B<unix> is used when serving requests coming from command line or
in inetd mode. See the section B<FORMAT>, for a detailed discussion.
+=item B<--export>
+
+Export the IP database into portable ASCII dump file. If a singe argument
+is supplied, it gives the name of the output file. In the absense of
+arguments, the output goes to the standard output.
+
+The created file can be transmitted over the network to hosts of another
+architecture and used there to recreate the database, using the
+B<whoseip --import>.
+
=item B<--fastcgi=>[I<SUFFIX...>]
When used without argument, forces FastCGI mode. If an argument is given,
@@ -1004,6 +1052,12 @@ Comments are introduced with a B<#> sign. Empty lines are ignored.
Without this option, B<whoseip> uses the built-in list of servers.
+=item B<--import>
+
+Import data from the file given as the first argument into the database. If
+no argument is given, read from standard input. The input must be a valid
+B<whoseip> database dump, as produced by B<whoseip --export>.
+
=item B<-N>, B<--no-cache>
Disable caching (this is the default).

Return to:

Send suggestions and report system problems to the System administrator.