diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2014-10-16 16:57:56 +0300 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2014-10-16 17:01:24 +0300 |
commit | 4005897a75d88355c6da8513bf7a4a13c301c97a (patch) | |
tree | e28fb6995ee2f995ef74cd44a09d5fb07419b495 | |
parent | b51b2856515a92fad170b91f9788798e288420ef (diff) | |
download | dnstools-4005897a75d88355c6da8513bf7a4a13c301c97a.tar.gz dnstools-4005897a75d88355c6da8513bf7a4a13c301c97a.tar.bz2 |
whoseip: fix caching algorithm
* whoseip/Whoseip/DB.pm: Fix caching algorithm, improve debugging.
Close all open databases before terminating.
* whoseip/whoseip.pl: Implement ${source} and ${item} macro variables.
Document new options and variables.
-rw-r--r-- | whoseip/Whoseip/DB.pm | 224 | ||||
-rw-r--r-- | whoseip/whoseip.pl | 33 |
2 files changed, 197 insertions, 60 deletions
diff --git a/whoseip/Whoseip/DB.pm b/whoseip/Whoseip/DB.pm index 0abc953..f6b08f0 100644 --- a/whoseip/Whoseip/DB.pm +++ b/whoseip/Whoseip/DB.pm | |||
@@ -160,8 +160,27 @@ use constant IPDB_PAGE_LEAF => 2; | |||
160 | 160 | ||
161 | use constant LEAF_IDX => 256; | 161 | use constant LEAF_IDX => 256; |
162 | 162 | ||
163 | sub pagetypestr { | ||
164 | my $t = shift; | ||
165 | return "index" if ($t == IPDB_PAGE_INDEX); | ||
166 | return "leaf" if ($t == IPDB_PAGE_LEAF); | ||
167 | return $t; | ||
168 | } | ||
169 | |||
163 | sub systell { sysseek($_[0], 0, SEEK_CUR) } | 170 | sub systell { sysseek($_[0], 0, SEEK_CUR) } |
164 | 171 | ||
172 | my @ipdb_open_files; | ||
173 | |||
174 | sub ipdb_close_all { | ||
175 | foreach my $file (@ipdb_open_files) { | ||
176 | ipdb_close($file) if defined($file->{fd}); | ||
177 | } | ||
178 | } | ||
179 | |||
180 | END { | ||
181 | ipdb_close_all(); | ||
182 | } | ||
183 | |||
165 | =pod | 184 | =pod |
166 | 185 | ||
167 | =head2 B<I<$dbf> = Whoseip::DB::ipdb_open(I<$filename>>[B<,> I<options>]B<);> | 186 | =head2 B<I<$dbf> = Whoseip::DB::ipdb_open(I<$filename>>[B<,> I<options>]B<);> |
@@ -221,7 +240,7 @@ sub ipdb_open { | |||
221 | for (my $i = 0; $i < $count; $i += 2) { | 240 | for (my $i = 0; $i < $count; $i += 2) { |
222 | $ipdbfile{rootidx}->{$tab[$i]} = $tab[$i+1]; | 241 | $ipdbfile{rootidx}->{$tab[$i]} = $tab[$i+1]; |
223 | print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n" | 242 | print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n" |
224 | if $_{debug}; | 243 | if $_{debug} > 1; |
225 | } | 244 | } |
226 | } else { | 245 | } else { |
227 | open($fd, "+>", $filename) | 246 | open($fd, "+>", $filename) |
@@ -241,10 +260,11 @@ sub ipdb_open { | |||
241 | $ipdbfile{maxpagecache} = | 260 | $ipdbfile{maxpagecache} = |
242 | defined($_{maxpagecache}) ? $_{maxpagecache} : 16; | 261 | defined($_{maxpagecache}) ? $_{maxpagecache} : 16; |
243 | $ipdbfile{debug} = $_{debug}; | 262 | $ipdbfile{debug} = $_{debug}; |
244 | if ($ipdbfile{debug}) { | 263 | if ($ipdbfile{debug} > 1) { |
245 | my $ug = new Data::UUID; | 264 | my $ug = new Data::UUID; |
246 | print STDERR "file $filename, UUID ".$ug->to_string($ipdbfile{uuid})."\n"; | 265 | print STDERR "file $filename, UUID ".$ug->to_string($ipdbfile{uuid})."\n"; |
247 | } | 266 | } |
267 | push @ipdb_open_files, \%ipdbfile; | ||
248 | return \%ipdbfile; | 268 | return \%ipdbfile; |
249 | } | 269 | } |
250 | 270 | ||
@@ -326,7 +346,7 @@ sub ipdb_locker { | |||
326 | for (my $i = 0; $i < $count; $i += 2) { | 346 | for (my $i = 0; $i < $count; $i += 2) { |
327 | $dbf->{rootidx}{$tab[$i]} = $tab[$i+1]; | 347 | $dbf->{rootidx}{$tab[$i]} = $tab[$i+1]; |
328 | print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n" | 348 | print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n" |
329 | if $_{debug}; | 349 | if $_{debug} > 1; |
330 | } | 350 | } |
331 | 351 | ||
332 | # Invalidate the cache | 352 | # Invalidate the cache |
@@ -347,47 +367,67 @@ sub ipdb_save_page($$) { | |||
347 | if ($page->{type} == IPDB_PAGE_INDEX) { | 367 | if ($page->{type} == IPDB_PAGE_INDEX) { |
348 | print STDERR "saving index page $page->{off}: ". | 368 | print STDERR "saving index page $page->{off}: ". |
349 | join(',', @{$page->{tab}})."\n" | 369 | join(',', @{$page->{tab}})."\n" |
350 | if $dbf->{debug}; | 370 | if $dbf->{debug} > 1; |
351 | $ret = syswrite($dbf->{fd}, pack('LL[257].', | 371 | $ret = syswrite($dbf->{fd}, pack('LL[257].', |
352 | $page->{type}, | 372 | $page->{type}, |
353 | @{$page->{tab}}, | 373 | @{$page->{tab}}, |
354 | $dbf->{pagesize}), | 374 | $dbf->{pagesize}), |
355 | $dbf->{pagesize}); | 375 | $dbf->{pagesize}); |
376 | croak "$dbf->{file}: write error at $page->{off}: $ret: $!" | ||
377 | unless ($ret == $dbf->{pagesize}); | ||
378 | |||
379 | delete $page->{dirty}; | ||
356 | } elsif ($page->{type} == IPDB_PAGE_LEAF) { | 380 | } elsif ($page->{type} == IPDB_PAGE_LEAF) { |
357 | print STDERR "saving leaf page $page->{off}\n" | 381 | my $nextpage; |
358 | if $dbf->{debug}; | 382 | do { |
359 | my @a; | 383 | print STDERR "saving leaf page $page->{off}\n" |
360 | my $size = length(pack('LLL',0,0,0)); | 384 | if $dbf->{debug} > 1; |
361 | my $i = 0; | 385 | my $size = length(pack('LLL',0,0,0)); |
362 | foreach my $ent (@{$page->{tab}}) { | 386 | my $i = 0; |
363 | my $x = pack('LLLa2L/a', @{$ent}[0 .. 3],freeze($ent->[4])); | 387 | my $a; |
364 | my $l = length($x); | 388 | foreach my $ent (@{$page->{tab}}) { |
365 | if ($size + $l > $dbf->{pagesize}) { | 389 | my $fdata = eval { freeze($ent->[4]) }; |
366 | my $p = ipdb_alloc_page($dbf, IPDB_PAGE_LEAF); | 390 | if ($@) { |
367 | $page->{next} = $p->{off}; | 391 | print STDERR "failed to freeze data for " . |
368 | $p->{tab} = @{$page->{tab}}[$i .. $#{$page->{tab}}]; | 392 | inet_ntoa(pack('N', $ent->[0])) . "/" . |
369 | $p->{dirty} = 1; | 393 | inet_ntoa(pack('N', $ent->[1])) . ":". |
370 | splice @{$page->{tab}}, $i; | 394 | $ent->[3] ."\n"; |
371 | last; | 395 | exit; |
396 | } | ||
397 | my $x = pack('LLLa2L/a', @{$ent}[0 .. 3],$fdata); | ||
398 | my $l = length($x); | ||
399 | if ($size + $l > $dbf->{pagesize}) { | ||
400 | print STDERR "SPLIT at $i: $size + $l, rest ". | ||
401 | ($#{$page->{tab}}-$i+1)."\n" | ||
402 | if $dbf->{debug} > 1; | ||
403 | $nextpage = ipdb_alloc_page($dbf, IPDB_PAGE_LEAF, | ||
404 | nocache => 1); | ||
405 | $page->{next} = $nextpage->{off}; | ||
406 | @{$nextpage->{tab}} = @{$page->{tab}}[$i .. $#{$page->{tab}}]; | ||
407 | $nextpage->{dirty} = 1; | ||
408 | splice @{$page->{tab}}, $i; | ||
409 | last; | ||
410 | } | ||
411 | $size += $l; | ||
412 | $a .= $x; | ||
413 | ++$i; | ||
372 | } | 414 | } |
373 | $size += $l; | 415 | $ret = syswrite($dbf->{fd}, |
374 | push @a, $x; | 416 | pack('LLLa'.length($a).'@'.$dbf->{pagesize}, |
375 | } continue { | 417 | $page->{type}, |
376 | ++$i; | 418 | $i, |
377 | } | 419 | $page->{next}, |
378 | $ret = syswrite($dbf->{fd}, | 420 | $a)); |
379 | pack('LLLa*@'.$dbf->{pagesize}, | 421 | croak "$dbf->{file}: write error at $page->{off}: $ret: $!" |
380 | $page->{type}, | 422 | unless ($ret == $dbf->{pagesize}); |
381 | $#a + 1, | 423 | delete $page->{dirty}; |
382 | $page->{next}, | 424 | $page = $nextpage; |
383 | @a)); | 425 | $nextpage = undef; |
426 | } while (defined($page)); | ||
427 | |||
384 | } else { | 428 | } else { |
385 | croak "unrecognized page type ($page->{type})"; | 429 | croak "unrecognized page type ($page->{type})"; |
386 | } | 430 | } |
387 | croak "$dbf->{file}: write error at $page->{off}: $ret: $!" | ||
388 | unless ($ret == $dbf->{pagesize}); | ||
389 | |||
390 | delete $page->{dirty}; | ||
391 | } | 431 | } |
392 | 432 | ||
393 | sub ipdb_cache_invalidate($) { | 433 | sub ipdb_cache_invalidate($) { |
@@ -414,26 +454,70 @@ sub ipdb_cache_put($$) { | |||
414 | $page->{lru_newer} = undef; | 454 | $page->{lru_newer} = undef; |
415 | $page->{lru_older} = $n; | 455 | $page->{lru_older} = $n; |
416 | $dbf->{pagecache}{lru_newest} = $page; | 456 | $dbf->{pagecache}{lru_newest} = $page; |
457 | $dbf->{pagecache}{lru_oldest} = $page | ||
458 | unless defined $dbf->{pagecache}{lru_oldest}; | ||
417 | $dbf->{pagecache}{$page->{off}} = $page; | 459 | $dbf->{pagecache}{$page->{off}} = $page; |
460 | dump_lru($dbf, "put $page->{off}"); | ||
418 | } | 461 | } |
419 | 462 | ||
463 | sub dump_lru { | ||
464 | my ($dbf,$pfx) = @_; | ||
465 | |||
466 | return unless $dbf->{debug} > 2; | ||
467 | |||
468 | my $x = $dbf->{pagecache}{lru_oldest}; | ||
469 | print STDERR "DUMP $pfx\n"; | ||
470 | print STDERR "KEYS: ".join(',', sort keys %{$dbf->{pagecache}})."\n"; | ||
471 | while (defined($x)) { | ||
472 | print STDERR "==> $x->{off} (".pagetypestr($x->{type}).","; | ||
473 | |||
474 | if (defined($x->{lru_newer})) { | ||
475 | print STDERR $x->{lru_newer}{off}; | ||
476 | } else { | ||
477 | print STDERR "NIL"; | ||
478 | } | ||
479 | print STDERR ","; | ||
480 | if (defined($x->{lru_older})) { | ||
481 | print STDERR $x->{lru_older}{off}; | ||
482 | } else { | ||
483 | print STDERR "NIL"; | ||
484 | } | ||
485 | print STDERR ")\n"; | ||
486 | $x = $x->{lru_newer}; | ||
487 | } | ||
488 | print STDERR "END\n"; | ||
489 | } | ||
490 | |||
420 | sub ipdb_cache_get($$) { | 491 | sub ipdb_cache_get($$) { |
421 | my ($dbf,$off) = @_; | 492 | my ($dbf,$off) = @_; |
422 | my $page; | 493 | my $page; |
423 | if (defined($dbf->{pagecache}{$off})) { | 494 | if (defined($dbf->{pagecache}{$off})) { |
495 | print STDERR "$off found in cache\n" if $dbf->{debug}; | ||
424 | $page = $dbf->{pagecache}{$off}; | 496 | $page = $dbf->{pagecache}{$off}; |
425 | # promote the page | ||
426 | if (defined($page->{lru_older})) { | ||
427 | $page->{lru_older}{lru_newer} = $page->{lru_newer}; | ||
428 | } else { | ||
429 | # It was the oldest page | ||
430 | $dbf->{pagecache}{lru_oldest} = $page->{lru_newer}; | ||
431 | } | ||
432 | if (defined($page->{lru_newer})) { |