aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org.ua>2014-10-16 16:57:56 +0300
committerSergey Poznyakoff <gray@gnu.org.ua>2014-10-16 17:01:24 +0300
commit4005897a75d88355c6da8513bf7a4a13c301c97a (patch)
treee28fb6995ee2f995ef74cd44a09d5fb07419b495
parentb51b2856515a92fad170b91f9788798e288420ef (diff)
downloaddnstools-4005897a75d88355c6da8513bf7a4a13c301c97a.tar.gz
dnstools-4005897a75d88355c6da8513bf7a4a13c301c97a.tar.bz2
whoseip: fix caching algorithm
* whoseip/Whoseip/DB.pm: Fix caching algorithm, improve debugging. Close all open databases before terminating. * whoseip/whoseip.pl: Implement ${source} and ${item} macro variables. Document new options and variables.
-rw-r--r--whoseip/Whoseip/DB.pm224
-rw-r--r--whoseip/whoseip.pl33
2 files changed, 197 insertions, 60 deletions
diff --git a/whoseip/Whoseip/DB.pm b/whoseip/Whoseip/DB.pm
index 0abc953..f6b08f0 100644
--- a/whoseip/Whoseip/DB.pm
+++ b/whoseip/Whoseip/DB.pm
@@ -160,8 +160,27 @@ use constant IPDB_PAGE_LEAF => 2;
160 160
161use constant LEAF_IDX => 256; 161use constant LEAF_IDX => 256;
162 162
163sub pagetypestr {
164 my $t = shift;
165 return "index" if ($t == IPDB_PAGE_INDEX);
166 return "leaf" if ($t == IPDB_PAGE_LEAF);
167 return $t;
168}
169
163sub systell { sysseek($_[0], 0, SEEK_CUR) } 170sub systell { sysseek($_[0], 0, SEEK_CUR) }
164 171
172my @ipdb_open_files;
173
174sub ipdb_close_all {
175 foreach my $file (@ipdb_open_files) {
176 ipdb_close($file) if defined($file->{fd});
177 }
178}
179
180END {
181 ipdb_close_all();
182}
183
165=pod 184=pod
166 185
167=head2 B<I<$dbf> = Whoseip::DB::ipdb_open(I<$filename>>[B<,> I<options>]B<);> 186=head2 B<I<$dbf> = Whoseip::DB::ipdb_open(I<$filename>>[B<,> I<options>]B<);>
@@ -221,7 +240,7 @@ sub ipdb_open {
221 for (my $i = 0; $i < $count; $i += 2) { 240 for (my $i = 0; $i < $count; $i += 2) {
222 $ipdbfile{rootidx}->{$tab[$i]} = $tab[$i+1]; 241 $ipdbfile{rootidx}->{$tab[$i]} = $tab[$i+1];
223 print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n" 242 print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n"
224 if $_{debug}; 243 if $_{debug} > 1;
225 } 244 }
226 } else { 245 } else {
227 open($fd, "+>", $filename) 246 open($fd, "+>", $filename)
@@ -241,10 +260,11 @@ sub ipdb_open {
241 $ipdbfile{maxpagecache} = 260 $ipdbfile{maxpagecache} =
242 defined($_{maxpagecache}) ? $_{maxpagecache} : 16; 261 defined($_{maxpagecache}) ? $_{maxpagecache} : 16;
243 $ipdbfile{debug} = $_{debug}; 262 $ipdbfile{debug} = $_{debug};
244 if ($ipdbfile{debug}) { 263 if ($ipdbfile{debug} > 1) {
245 my $ug = new Data::UUID; 264 my $ug = new Data::UUID;
246 print STDERR "file $filename, UUID ".$ug->to_string($ipdbfile{uuid})."\n"; 265 print STDERR "file $filename, UUID ".$ug->to_string($ipdbfile{uuid})."\n";
247 } 266 }
267 push @ipdb_open_files, \%ipdbfile;
248 return \%ipdbfile; 268 return \%ipdbfile;
249} 269}
250 270
@@ -326,7 +346,7 @@ sub ipdb_locker {
326 for (my $i = 0; $i < $count; $i += 2) { 346 for (my $i = 0; $i < $count; $i += 2) {
327 $dbf->{rootidx}{$tab[$i]} = $tab[$i+1]; 347 $dbf->{rootidx}{$tab[$i]} = $tab[$i+1];
328 print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n" 348 print STDERR "ROOTIDX $tab[$i]=$tab[$i+1]\n"
329 if $_{debug}; 349 if $_{debug} > 1;
330 } 350 }
331 351
332 # Invalidate the cache 352 # Invalidate the cache
@@ -347,47 +367,67 @@ sub ipdb_save_page($$) {
347 if ($page->{type} == IPDB_PAGE_INDEX) { 367 if ($page->{type} == IPDB_PAGE_INDEX) {
348 print STDERR "saving index page $page->{off}: ". 368 print STDERR "saving index page $page->{off}: ".
349 join(',', @{$page->{tab}})."\n" 369 join(',', @{$page->{tab}})."\n"
350 if $dbf->{debug}; 370 if $dbf->{debug} > 1;
351 $ret = syswrite($dbf->{fd}, pack('LL[257].', 371 $ret = syswrite($dbf->{fd}, pack('LL[257].',
352 $page->{type}, 372 $page->{type},
353 @{$page->{tab}}, 373 @{$page->{tab}},
354 $dbf->{pagesize}), 374 $dbf->{pagesize}),
355 $dbf->{pagesize}); 375 $dbf->{pagesize});
376 croak "$dbf->{file}: write error at $page->{off}: $ret: $!"
377 unless ($ret == $dbf->{pagesize});
378
379 delete $page->{dirty};
356 } elsif ($page->{type} == IPDB_PAGE_LEAF) { 380 } elsif ($page->{type} == IPDB_PAGE_LEAF) {
357 print STDERR "saving leaf page $page->{off}\n" 381 my $nextpage;
358 if $dbf->{debug}; 382 do {
359 my @a; 383 print STDERR "saving leaf page $page->{off}\n"
360 my $size = length(pack('LLL',0,0,0)); 384 if $dbf->{debug} > 1;
361 my $i = 0; 385 my $size = length(pack('LLL',0,0,0));
362 foreach my $ent (@{$page->{tab}}) { 386 my $i = 0;
363 my $x = pack('LLLa2L/a', @{$ent}[0 .. 3],freeze($ent->[4])); 387 my $a;
364 my $l = length($x); 388 foreach my $ent (@{$page->{tab}}) {
365 if ($size + $l > $dbf->{pagesize}) { 389 my $fdata = eval { freeze($ent->[4]) };
366 my $p = ipdb_alloc_page($dbf, IPDB_PAGE_LEAF); 390 if ($@) {
367 $page->{next} = $p->{off}; 391 print STDERR "failed to freeze data for " .
368 $p->{tab} = @{$page->{tab}}[$i .. $#{$page->{tab}}]; 392 inet_ntoa(pack('N', $ent->[0])) . "/" .
369 $p->{dirty} = 1; 393 inet_ntoa(pack('N', $ent->[1])) . ":".
370 splice @{$page->{tab}}, $i; 394 $ent->[3] ."\n";
371 last; 395 exit;
396 }
397 my $x = pack('LLLa2L/a', @{$ent}[0 .. 3],$fdata);
398 my $l = length($x);
399 if ($size + $l > $dbf->{pagesize}) {
400 print STDERR "SPLIT at $i: $size + $l, rest ".
401 ($#{$page->{tab}}-$i+1)."\n"
402 if $dbf->{debug} > 1;
403 $nextpage = ipdb_alloc_page($dbf, IPDB_PAGE_LEAF,
404 nocache => 1);
405 $page->{next} = $nextpage->{off};
406 @{$nextpage->{tab}} = @{$page->{tab}}[$i .. $#{$page->{tab}}];
407 $nextpage->{dirty} = 1;
408 splice @{$page->{tab}}, $i;
409 last;
410 }
411 $size += $l;
412 $a .= $x;
413 ++$i;
372 } 414 }
373 $size += $l; 415 $ret = syswrite($dbf->{fd},
374 push @a, $x; 416 pack('LLLa'.length($a).'@'.$dbf->{pagesize},
375 } continue { 417 $page->{type},
376 ++$i; 418 $i,
377 } 419 $page->{next},
378 $ret = syswrite($dbf->{fd}, 420 $a));
379 pack('LLLa*@'.$dbf->{pagesize}, 421 croak "$dbf->{file}: write error at $page->{off}: $ret: $!"
380 $page->{type}, 422 unless ($ret == $dbf->{pagesize});
381 $#a + 1, 423 delete $page->{dirty};
382 $page->{next}, 424 $page = $nextpage;
383 @a)); 425 $nextpage = undef;
426 } while (defined($page));
427
384 } else { 428 } else {
385 croak "unrecognized page type ($page->{type})"; 429 croak "unrecognized page type ($page->{type})";
386 } 430 }
387 croak "$dbf->{file}: write error at $page->{off}: $ret: $!"
388 unless ($ret == $dbf->{pagesize});
389
390 delete $page->{dirty};
391} 431}
392 432
393sub ipdb_cache_invalidate($) { 433sub ipdb_cache_invalidate($) {
@@ -414,26 +454,70 @@ sub ipdb_cache_put($$) {
414 $page->{lru_newer} = undef; 454 $page->{lru_newer} = undef;
415 $page->{lru_older} = $n; 455 $page->{lru_older} = $n;
416 $dbf->{pagecache}{lru_newest} = $page; 456 $dbf->{pagecache}{lru_newest} = $page;
457 $dbf->{pagecache}{lru_oldest} = $page
458 unless defined $dbf->{pagecache}{lru_oldest};
417 $dbf->{pagecache}{$page->{off}} = $page; 459 $dbf->{pagecache}{$page->{off}} = $page;
460 dump_lru($dbf, "put $page->{off}");
418} 461}
419 462
463sub dump_lru {
464 my ($dbf,$pfx) = @_;
465
466 return unless $dbf->{debug} > 2;
467
468 my $x = $dbf->{pagecache}{lru_oldest};
469 print STDERR "DUMP $pfx\n";
470 print STDERR "KEYS: ".join(',', sort keys %{$dbf->{pagecache}})."\n";
471 while (defined($x)) {
472 print STDERR "==> $x->{off} (".pagetypestr($x->{type}).",";
473
474 if (defined($x->{lru_newer})) {
475 print STDERR $x->{lru_newer}{off};
476 } else {
477 print STDERR "NIL";
478 }
479 print STDERR ",";
480 if (defined($x->{lru_older})) {
481 print STDERR $x->{lru_older}{off};
482 } else {
483 print STDERR "NIL";
484 }
485 print STDERR ")\n";
486 $x = $x->{lru_newer};
487 }
488 print STDERR "END\n";
489}
490
420sub ipdb_cache_get($$) { 491sub ipdb_cache_get($$) {
421 my ($dbf,$off) = @_; 492 my ($dbf,$off) = @_;
422 my $page; 493 my $page;
423 if (defined($dbf->{pagecache}{$off})) { 494 if (defined($dbf->{pagecache}{$off})) {
495 print STDERR "$off found in cache\n" if $dbf->{debug};
424 $page = $dbf->{pagecache}{$off}; 496 $page = $dbf->{pagecache}{$off};
425 # promote the page
426 if (defined($page->{lru_older})) {
427 $page->{lru_older}{lru_newer} = $page->{lru_newer};
428 } else {
429 # It was the oldest page
430 $dbf->{pagecache}{lru_oldest} = $page->{lru_newer};
431 }
432 if (defined($page->{lru_newer})) {