diff options
author | Sergey Poznyakoff <gray@gnu.org.ua> | 2017-09-06 09:22:23 +0200 |
---|---|---|
committer | Sergey Poznyakoff <gray@gnu.org.ua> | 2017-09-06 09:22:23 +0200 |
commit | 3fd2e78286b5920c386652f9660282b6e97866ce (patch) | |
tree | fff2d0f2d3df22889a2bebf9d9a1e2da871b5e36 | |
parent | 2fe0a2ed10fbdde78655cf93296b343acfa78d7d (diff) | |
download | savane-gray-3fd2e78286b5920c386652f9660282b6e97866ce.tar.gz savane-gray-3fd2e78286b5920c386652f9660282b6e97866ce.tar.bz2 |
Remove sv_usrck. It's been integrated to sv_spamchecker
-rwxr-xr-x | backend/accounts/sv_usrck | 585 |
1 files changed, 0 insertions, 585 deletions
diff --git a/backend/accounts/sv_usrck b/backend/accounts/sv_usrck deleted file mode 100755 index d295508..0000000 --- a/backend/accounts/sv_usrck +++ /dev/null @@ -1,585 +0,0 @@ -#! /usr/bin/perl -# Copyright (C) 2015-2017 Sergey Poznyakoff <gray@gnu.org> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -use strict; -use Savane; -use Savane::DB; -use Savane::Backend; -use Savane::Trackers; -use Mail::SpamAssassin::Client; -use Data::Dumper; -use File::Temp qw(tempfile); -use Time::ParseDate; -use Data::Dumper; - -use constant TYPE_SPAM => 0; -use constant TYPE_HAM => 1; -use constant TYPE_FORGET => 2; - -my $spamc; -my $repfd; -my $report_detail; -my $min_score = 5; -my $quiet; -my $learn; -my $assume_status; -my $since; -my $sort; -my $update_spamscore; -my $reporter_uid = 101; # FIXME - -my %all_trackers = map { $_ => 1 } TrackerNames(); -my $enable_resume; -my @enable_trackers; - -my $sys_spamc_socketpath = GetConf('backend.sv_spamchecker.socket'); -my $sys_spamc_host = GetConf('backend.sv_spamchecker.host'); -my $sys_spamc_port = GetConf('backend.sv_spamchecker.port'); -my $sys_spamc_timeout = GetConf('backend.sv_spamchecker.timeout'); -my $sys_spamc_username = GetConf('backend.sv_spamchecker.user'); -my $sys_https_host = GetConf('core.domain'); - -sub userlist_collect { - my @userlist; - my $cond; - - if ($#_ == -1) { - $cond = "user.status='A' AND user_group.group_id IS NULL"; - } else { - if ($#_ == 0) { - $cond = "user.user_name=?"; - } else { - $cond = "user.user_name IN (" . join(',', ('?') x (@_)) . ')'; - } - } - if ($since) { - $cond .= qq{ AND user.add_date >= $since}; - } - - my $order = ''; - if ($sort) { - $order = 'ORDER BY user_name'; - } - - db_foreach(sub { push @userlist, $_[0] }, - qq{ -SELECT distinct user.user_id AS user_id, - user.user_name AS user_name, - user.realname AS realname, - user.people_resume AS resume, - user.people_view_skills AS view_skills, - user.add_date AS date, - cache.isspam AS cache_isspam -FROM user -LEFT JOIN user_group ON user_group.user_id=user.user_id -LEFT JOIN trackers_spamcheck_cache cache - ON cache.item_id = user.user_id - AND cache.artifact = 'resume' -WHERE $cond -$order - }, - @_); - return @userlist; -} - -# sub check_cache { -# my ($ref, $ctr) = @_; -# if (defined($ref->{cache_isspam})) { -# } -# } - -sub spam_check { - my ($msg, $ctr, $tracker, $id) = @_; - my $result; - - $ctr->{total}++; - if (defined($assume_status)) { - return { isspam => $assume_status == TYPE_SPAM ? 'True' : 'False' }; - } - - if (defined($report_detail)) { - $result = $spamc->_filter($msg, $report_detail); - } else { - $result = $spamc->process($msg); - } - - push @{$ctr->{detail}}, - { tracker => $tracker, item_id => $id, msg => $msg, - result => $result } - if $repfd; - - $ctr->{sum} += $result->{score}; - if ($result->{isspam} eq 'True') { - ++$ctr->{spam}; - } else { - ++$ctr->{ham}; - } - return $result; -} - -sub update_trackers_spamscore { - my ($newscore, $tracker, $user_id, $bug_id, $comment_id) = @_; - # Feed the spamscore table -# db_delete('trackers_spamscore', -# artifact => $tracker, -# item_id => $ref->{bug_id}, -# comment_id => $ref->{comment_id}); - db_modify(q{ -DELETE FROM trackers_spamscore -WHERE artifact=? AND item_id=? AND comment_id=? AND affected_user_id=? -}, - $tracker, $bug_id, $comment_id, $user_id); - db_insert('trackers_spamscore', - score => $newscore, - affected_user_id => $user_id, - reporter_user_id => $reporter_uid, - artifact => $tracker, - item_id => $bug_id, - comment_id => $comment_id); - - my $spam = $newscore > 4 ? 'Y' : 'N'; - db_modify(qq{ -INSERT INTO trackers_spamcheck_cache (artifact, item_id, comment_id, isspam) -VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE isspam=? - }, $tracker, $bug_id, $comment_id, $spam, $spam); -} - -sub check_tracker_comment { - my ($tracker, $ref, $ctr, $user, $cb) = @_; - # FIXME: cache - my $msg = GetTrackersContentAsMail($user->{user_id}, - $ref->{ip}, - $tracker, - $ref->{bug_id}, - $ref->{comment_id}, - $ref->{date}, - $ref->{summary}, - $ref->{details}); - my $res = spam_check($msg, $ctr, $tracker, $ref->{bug_id}); - - &{$cb} ($msg, $res, $user->{user_name}, "$tracker $ref->{bug_id}") - if (ref($cb) eq 'CODE'); - - if ($update_spamscore) { - my $newscore = savane_adjust_spamscore($ref->{spamscore}, - $res->{score}); - my ($q,$id); - if ($ref->{comment_id} > 0) { - $q = qq{UPDATE ${tracker}_history SET spamscore=? WHERE bug_history_id=?}; - $id = $ref->{comment_id}; - } else { - $q = qq{UPDATE $tracker SET spamscore=? WHERE bug_id=?}; - $id = $ref->{bug_id}; - } - db_modify($q, $newscore, $id); - update_trackers_spamscore($newscore, $tracker, - $user->{user_id}, - $ref->{bug_id}, $ref->{comment_id}); - } -} - -sub general_tracker_check { - my ($tracker,$uref,$ctr,$cb) = @_; - my $uid = $uref->{user_id}; - - db_foreach(sub { - my $ref = shift; - $ref->{comment_id} = 0; - check_tracker_comment($tracker, $ref, $ctr, $uref, $cb); - }, - qq{ -SELECT tracker.bug_id AS bug_id, - tracker.summary AS summary, - tracker.details AS details, - tracker.date AS date, - tracker.ip AS ip, - tracker.spamscore AS spamscore, - cache.isspam AS cache_isspam -FROM $tracker tracker -LEFT JOIN trackers_spamcheck_cache cache - ON tracker.bug_id = cache.item_id - AND cache.artifact = ? -WHERE tracker.submitted_by=? -ORDER BY t.date - }, $tracker, $uid); - - db_foreach(sub { - my $ref = shift; - check_tracker_comment($tracker, $ref, $ctr, $uref, $cb); - }, - qq{ -SELECT h.bug_history_id AS comment_id, - h.bug_id AS bug_id, - h.old_value AS details, - h.ip AS ip, - h.date AS date, - h.spamscore AS spamscore, - t.summary AS summary, - cache.isspam AS isspam -FROM ${tracker}_history h, $tracker t -LEFT JOIN trackers_spamcheck_cache cache - ON h.bug_id = cache.item_id - AND cache.artifact = ? -WHERE field_name='details' AND mod_by=? AND t.bug_id=h.bug_id -ORDER BY date}, - $tracker, $uid); - -} - -sub resume_check { - return unless $enable_resume; - # FIXME: cache - my $msg = GetTrackersContentAsMail($uref->{user_id}, - '127.0.0.1', - 'resume', - $uref->{user_id}, # bug_id - '0', # comment_id - $uref->{date}, - 'Resume', - $uref->{resume}); - my $res = spam_check($msg, $ctr, 'resume', 0); - &{$cb} ($msg, $res, $uref->{user_name}, 'Resume') if (ref($cb) eq 'CODE'); - - if ($update_spamscore) { - my $newscore = savane_adjust_spamscore(0, $res->{score}); - db_modify(q{UPDATE user SET resume_spamscore=? WHERE user_id=?}, - $newscore, $uref->{user_id}); - update_trackers_spamscore($newscore, 'resume', - $uref->{user_id}, 0, 0); - } -} - -sub result_is_spam { - my $result = shift; - if (defined($min_score)) { - return $result->{score} >= $min_score; - } else { - return $result->{isspam} eq 'True'; - } -} - -sub report_uid { - my ($usr, $ctr) = @_; - print $repfd "* $usr->{user_name}: $ctr->{status}\n"; - print $repfd "total=$ctr->{total}\n"; - print $repfd "score=$ctr->{score}\n"; - print $repfd "spam=$ctr->{spam} / $ctr->{ham}\n"; - - return unless defined $ctr->{detail}; - - my $tracker; - foreach my $detail (@{$ctr->{detail}}) { - print $repfd "\n"; - - my $result = $detail->{result}; - - if ($tracker ne $detail->{tracker}) { - $tracker = $detail->{tracker}; - print $repfd "\n** $tracker\n"; - } - - my $stat = $result->{isspam} eq 'True'; - print $repfd "\n*** $detail->{item_id}: $stat (score $result->{score})\n"; - my $print_msg; - - if (exists($result->{message})) { - $result->{message} =~ s/.*^(Content analysis details:)/$1/sm; - print $repfd "$result->{message}\n\n"; - $print_msg = 1 if $report_detail; - } else { - $print_msg = 1; - } - if ($print_msg) { - $detail->{msg} =~ s/\r//g; - $detail->{msg} =~ s/^\*/\\*/; - print $repfd "$detail->{msg}\n"; - } - } - print $repfd "\n\f\n"; -} - -sub parsetime { - my $s = shift; - my $t = 0; - if ($s =~ /^\d+$/) { - $t = $s; - } else { - my @keyord = ('Y', 'M', 'd', 'h', 'm', 's'); - my %timetrans = ('Y' => 365*86400, - 'M' => 31*86400, - 'd' => 86400, - 'h' => 3600, - 'm' => 60, - 's' => 1); - for (my $set = join('', @keyord); - $set ne '' && $s =~ /(\d+)\s*([$set])\s*(.*)/; - $set = join('', @keyord)) { - while (shift(@keyord) ne $2) {} - $t += $1 * $timetrans{$2}; - $s = $3; - } - abend(EX_USAGE, "time specification error near $s") unless $s eq ''; - } - return $t; -} - -# #### -my $file; -my $report; -my %d2cmd = ( 1 => 'SYMBOLS', - symbols => 'SYMBOLS', - 2 => 'REPORT', - full => 'REPORT' ); - -my %opts = ("report|r" => \$report, - "detail|D=s" => sub { - if ($_[1] ne '0') { - my $s = lc $_[1]; - abend(EX_USAGE, "unknown detail level") - unless defined $d2cmd{$s}; - $report_detail = $d2cmd{$s}; - } - $report = 1; - }, - "output|o=s" => \$file, - "quiet|q" => \$quiet, - "score|s=f" => \$min_score, - "learn|l" => \$learn, - "update-spamscore" => \$update_spamscore, - "assume-status=s" => sub { - $assume_status = learntype($_[1]); - abend(EX_USAGE, "unrecognized status") - unless (defined($assume_status) && $assume_status != TYPE_FORGET); - }, - "resume" => \$enable_resume, - "no-resume" => sub { $enable_resume = 0 }, - "trackers=s" => sub { - foreach my $name (split /,/, $_[1]) { - push @enable_trackers, $name; - } - }, - "no-trackers:s" => sub { - if ($_[1] eq '') { - %all_trackers = (); - } else { - foreach my $name (split /,/, $_[1]) { - delete $all_trackers{$name}; - } - } - }, - "since:s" => sub { $since = parsedate($_[1]) }, - "sort" => \$sort - ); -backend_setup(descr => "Check active users", - cron => GetConf('backend.sv_usrck.cron')||1, # FIXME - options => \%opts); - -if (defined($assume_status)) { - abend(EX_USAGE, "--assume-status is valid only with --learn") - unless defined $learn; - abend(EX_USAGE, "--update-spamscore is not valid with --assume-status") - if $update_spamscore; -} - -$enable_resume = 1 unless defined $enable_resume; -@enable_trackers = sort keys %all_trackers - unless @enable_trackers; - -my %spamc_options; - -if (defined $sys_spamc_socketpath) { - $spamc_options{socketpath} = $sys_spamc_socketpath; -} elsif (defined $sys_spamc_port) { - $spamc_options{port} = $sys_spamc_port; - $spamc_options{host} = - defined($sys_spamc_host) ? $sys_spamc_host : 'localhost'; -} -$spamc_options{timeout} = $sys_spamc_timeout; -$spamc_options{username} = $sys_spamc_username; - -$spamc = new Mail::SpamAssassin::Client(\%spamc_options); -abend(EX_UNAVAILABLE, "can't connect to spamd") unless $spamc->ping(); - -if ($report) { - if (defined($file)) { - open($repfd, '>', $file) or - abend(EX_CANTCREAT, "can't create file $file"); - } else { - open($repfd, '>&', STDOUT); - ++$quiet; - } -} - -my $now = time(); - -# $s = savane_adjust_spamscore($input, $score); -# Adjusts Savane spamscore $input in accordance with the newly computed -# SpamAssassin $score. -sub savane_adjust_spamscore { - my ($spam_score, $sa_score) = @_; - my $newscore; - - if ($sa_score <= 0) { - $newscore = $spam_score - 8; - } elsif ($sa_score > 0.0 && $sa_score <= 2.0) { - $newscore = $spam_score - 5; - } elsif ($sa_score > 2.0 && $sa_score <= 3.0) { - $newscore = $spam_score - 4; - } elsif ($sa_score > 3.0 && $sa_score <= 5.0) { - $newscore = $spam_score - 3; - } elsif ($sa_score > 5.0 && $sa_score <= 7.0) { - $newscore = $spam_score - 2; - } elsif ($sa_score > 7.0 && $sa_score <= 9.0) { - $newscore = $spam_score - 1; - } else { - $newscore = $sa_score; - } - return $newscore < 0 ? 0 : int($newscore + 0.5); -} - -sub check_user { - my $uref = shift; - - my %ctr; - - $ctr{total} = $ctr{spam} = $ctr{ham} = 0; - if ($uref->{view_skills}) { - debug("$uref->{user_name}: checking resume"); - resume_check($uref, \%ctr); - } - - for my $t (@enable_trackers) { - debug("$uref->{user_name}: checking $t"); - general_tracker_check($t, $uref, \%ctr); - } - my $s = "$uref->{user_name} ($uref->{realname}): total=$ctr{total}"; - if ($ctr{total}) { - $ctr{score} = $ctr{sum} / $ctr{total}; - $ctr{status} = ($ctr{score} >= $min_score) ? 'SPAM' : 'OK'; - $s .= " avg score=$ctr{score}; spam=$ctr{spam}; ham=$ctr{ham}"; - } else { - $ctr{score} = 0.0; - $ctr{status} = 'OK'; - } - debug($s); - - print "$uref->{user_name} ($uref->{realname}): $ctr{status}\n" - unless $quiet; - report_uid($uref, \%ctr) if $report; - db_modify(q{UPDATE user SET spamscore=? WHERE user_id=?}, - savane_adjust_spamscore(8, $ctr{score}), $uref->{user_id}) - if $update_spamscore; -} - -sub line_count { - my $s = shift; - return 0 if $s eq ''; - my $count = 1; - $count++ while $s =~ m/$/g; - return $count; -} - -sub learntype { - my $s = uc shift; - my %t = ( - S => TYPE_SPAM, - SPAM => TYPE_SPAM, - H => TYPE_HAM, - HAM => TYPE_HAM. - F => TYPE_FORGET, - FORGET => TYPE_FORGET - ); - return $t{$s} if exists $t{$s}; - return undef; -} - -sub edit { - my $filename = shift; - my $editor = $ENV{'VISUAL'} || $ENV{'EDITOR'} || 'vi'; - system("$editor $filename"); - if ($? == -1) { - die "failed to execute $editor: $!"; - } elsif ($? & 127) { - die "$editor died with signal " . ($? & 127) . ' '. - (($? & 128) ? 'with' : 'without') . - ' coredump'; - } elsif ($? >> 8) { - die "$editor exited with code " . ($? >> 8); - } -} - -sub inspect { - my ($input, $result, $user, $what) = @_; - my ($learntype, $type); - - my ($fh, $filename) = tempfile(); - - $type = $result->{isspam} eq 'True' ? TYPE_SPAM : TYPE_HAM; - - print $fh "Status: ".($type == TYPE_SPAM ? "S" : "H")."\n"; - print $fh "User: $user\n"; - print $fh "Item: $what\n"; - print $fh "\n"; - print $fh $input; - close $fh; - - edit($filename); - - open(my $fh, '<', $filename) or - abend(EX_NOINPUT, "can't read temp file $filename: $!"); - while (<$fh>) { - chomp; - if (/^Status:\s*(.)/) { - $learntype = learntype($1); - last; - } - last if (/^\s*$/); - } - close $fh; - unlink $filename; - - if (defined($learntype) - && (defined($assume_status) || $learntype != $type)) { - print "Learned $learntype\n"; - my $res = $spamc->learn($input, $learntype); - abend(EX_UNAVAILABLE, $spamc->{resp_msg}) unless defined $res; - print "RES $res\n"; - } -} - -sub learn_user { - my $uref = shift; - print "Inspecting user ".$uref->{user_name}. "\n"; - my %ctr; - - if ($uref->{view_skills}) { - resume_check($uref, \%ctr, \&inspect); - } - - for my $t (@enable_trackers) { - debug("$uref->{user_name}: checking $t"); - general_tracker_check($t, $uref, \%ctr, \&inspect); - } -} - -my $action = defined($learn) ? \&learn_user : \&check_user; - -foreach my $uref (userlist_collect @ARGV) { - &{$action}($uref); -} - -backend_done; |