New utilities: sv_users_ldap and sv_groups_ldap

3 files changed, 631 insertions, 0 deletions

diff --git a/backend/Makefile.PL b/backend/Makefile.PL
index 43da7ec..fc7f72d 100644
--- a/backend/Makefile.PL
+++ b/backend/Makefile.PL
@@ -9,7 +9,9 @@ WriteMakefile(
     'VERSION'          => '1.0', # FIXME
     'EXE_FILES'        => [
 	'accounts/sv_groups', 
+	'accounts/sv_groups_ldap', 
 	'accounts/sv_users', 
+	'accounts/sv_users_ldap', 
 	'accounts/sv_membersh',
 	'accounts/sv_getpubkeys',
 	'accounts/sv_assign_uid_gid', 
diff --git a/backend/accounts/sv_groups_ldap b/backend/accounts/sv_groups_ldap
new file mode 100755
index 0000000..9a3b5a1
--- /dev/null
+++ b/backend/accounts/sv_groups_ldap
@@ -0,0 +1,481 @@
+#! /usr/bin/perl
+# Copyright (C) 2015, 2016 Sergey Poznyakoff <gray@gnu.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict;
+use Savane;
+use Savane::Conf;
+use Savane::Backend;
+use Savane::Feature qw(:prio);
+use Savane::LDAP;
+
+use Data::Dumper;
+
+my $sys_cron_groups = GetConf('backend.sv_groups.cron');
+my $sys_min_gid = GetConf('users.min_gid');
+my $sys_git_projects_file = GetConf('backend.sv_groups.project_list');
+my $sys_git_cgitrepos_file = GetConf('backend.sv_groups.cgitrepos_file');
+    
+my %select_type;
+
+# New-style initialization:
+backend_setup(descr => "Synchronize Savane group database with LDAP", 
+	      cron => $sys_cron_groups,
+	      options => {
+		  "type|t:s" => sub { $select_type{$_[1]} = 1; },
+	      });
+
+AcquireReplicationLock('groups-users.lock');
+
+# ref layout:
+#  { 'group_id' => '103',
+#    'group_name' => 'whatever',
+#    'type' => '2',
+#    'is_public' => '1',
+#    'unix_group_name' => 'group',
+#    'gidNumber' => '7017',
+#    'group_type_name' => 'Software',
+#    'group_type_homepage_scm' => 'cvs',
+#    'feature' => {
+#        'homepage' => {
+# 	   'dir_type' => 'basiccvs',
+# 	   'dir' => '/home/project/%PROJECT'
+#        },
+#        ...	       
+#    }
+#  }
+
+sub build_feature {
+    my ($feature, $ref) = @_;
+    return unless exists $ref->{feature}{$feature};
+    return if $ref->{feature}{$feature}{dir} eq '';
+
+    debug("updating $ref->{unix_group_name},$feature");
+    my $obj = backend_feature($feature, $ref);
+    return unless $obj;
+
+    $obj->build() and $obj->updatedb();
+
+    backend_feature_log($obj);
+    
+    return $obj;
+}
+
+my @features = ('homepage', 'download', 'cvs', 
+		'arch', 'svn', 'git', 'hg', 'bzr');
+
+my @group_columns = (
+    'group_id', 
+    'group_name',
+    'type', 
+    'is_public',
+    'unix_group_name',
+    'gidNumber'
+);
+
+my @type_columns = (
+    'name',
+    'homepage_scm'
+);
+
+my @select_columns = (
+    # First, group columns:
+    (map { "g.$_ as $_" } @group_columns),
+    (map { "g.use_$_ as use_$_" } @features),
+    (map { "g.dir_$_ as dir_$_" } @features),
+    # Now, group type columns:
+    (map { "t.$_ as group_type_$_" } @type_columns),
+    (map { "t.can_use_$_ as group_type_can_use_$_" } @features),
+    (map { "t.dir_$_ as group_type_dir_$_" } @features),
+    (map { "t.dir_type_$_ as dir_type_$_" } @features)
+);
+
+my $ldap = new Savane::LDAP;
+
+logit('info', "processing Savane groups");
+foreach my $ref (GetDBHashArray("groups g, group_type t", 
+				"g.status='A' AND g.type=t.type_id",
+				join(',', @select_columns))) {
+    next if keys %select_type > 0
+	and !exists($select_type{$ref->{group_type_name}});
+
+    foreach my $f (@features) {
+	$ref->{"use_$f"} = $ref->{"group_type_can_use_$f"}
+	    unless exists $ref->{"use_$f"};
+	if ($ref->{"use_$f"}) {
+	    $ref->{feature}{$f}{dir_type} = $ref->{"dir_type_$f"};
+	    $ref->{feature}{$f}{dir} =
+		(exists $ref->{"dir_$f"} && $ref->{"dir_$f"} ne '')
+		  ? $ref->{"dir_$f"}
+	          : $ref->{"group_type_dir_$f"};
+	}
+	delete $ref->{"use_$f"};
+	delete $ref->{"group_type_can_use_$f"};
+	delete $ref->{"dir_type_$f"};
+	delete $ref->{"dir_$f"};
+	delete $ref->{"group_type_dir_$f"};
+    }
+    
+    if (debug_level()) {
+	local $Data::Dumper::Indent = 0;
+	local $Data::Dumper::Terse = 1;
+	local $Data::Dumper::Purity = 0;
+	debug(Dumper($ref));
+    }
+
+    my $grp = $ldap->getgrnam($ref->{unix_group_name});
+    unless ($grp) {
+	debug("creating group $ref->{unix_group_name}");
+	$grp = new Savane::LDAP::grent(name => $ref->{unix_group_name},
+				       gid => $ldap->nextgid($sys_min_gid));
+	$ldap->chgrent($grp);
+    }
+
+    unless (defined($ref->{gidNumber}) && $ref->{gidNumber} == $grp->gid) {
+	debug("$ref->{unix_group_name}: updating GID in database: ".$grp->gid);
+	$ref->{gidNumber} = $grp->gid;
+	db_modify(qq{UPDATE groups SET gidNumber=? WHERE group_id=?},
+		  $grp->gid, $ref->{group_id}) unless dry_run();
+    }
+    
+    foreach my $f (@features) {
+	# Build the feature if needed.
+	build_feature($f, $ref);
+    }
+}
+
+logit('info', "updating git trees");
+
+my $updateinfo;
+foreach my $ref (GetDBHashArray("git_repo r,groups g,group_type t", 
+				"r.updated>r.synchronized and g.group_id=r.group_id and g.type=t.type_id",
+				"r.repo_id as repo_id,".
+				"g.group_id as group_id,".
+				"g.type as type,".
+				"g.is_public as is_public,".
+				"g.unix_group_name as unix_group_name,".
+				"g.gidNumber as gidNumber,".
+				"g.use_git as use_git,".
+				"g.dir_git as dir_git,".
+				"r.name as repo_name,".
+				"r.master as master,".
+				"r.owner as owner,".
+				"r.description as repo_short_description,".
+				"r.readme_html as repo_long_description,".
+				"t.name as group_type_name,".
+				"t.homepage_scm as group_type_homepage_scm,".
+				"t.dir_git as group_type_dir_git,".
+				"t.can_use_git as can_use_git,".
+				"t.dir_type_git as dir_type_git")) {
+    $ref->{use_git} = $ref->{can_use_git} unless $ref->{use_git};
+    next unless $ref->{use_git};
+    $ref->{feature}{git}{dir_type} = $ref->{dir_type_git};
+    $ref->{feature}{git}{dir} = $ref->{dir_git} || $ref->{group_type_dir_git};
+    
+    delete $ref->{dir_type_git};
+    delete $ref->{dir_git};
+    delete $ref->{group_type_dir_git};
+
+    if (debug_level()) {
+	local $Data::Dumper::Indent = 0;
+	local $Data::Dumper::Terse = 1;
+	local $Data::Dumper::Purity = 0;
+	debug(Dumper($ref));
+    }
+    
+    my $obj = build_feature('git', $ref);
+    if ($obj and $obj->success) {
+	$updateinfo = 1;
+	SetDBSettings("git_repo",
+		      'repo_id=?',
+		      'synchronized=now()', $ref->{repo_id});
+    }
+}
+
+if (($sys_git_projects_file && ! -e $sys_git_projects_file)
+    || ($sys_git_cgitrepos_file && ! -e $sys_git_cgitrepos_file)) {
+    $updateinfo = 1;
+}
+
+if ($updateinfo && ($sys_git_projects_file || $sys_git_cgitrepos_file)) {
+    my $f = $sys_git_projects_file;
+    if (defined($sys_git_cgitrepos_file)) {
+	$f .= ' and ' if defined($f);
+	$f .= $sys_git_cgitrepos_file;
+    }
+    logit('info', "updating $f");
+
+    my ($proj_fd, $cgit_fd);
+    if ($sys_git_projects_file) {
+	if (dry_run()) {
+	    open($proj_fd, ">&STDOUT");
+	} else {
+	    open($proj_fd, '>', $sys_git_projects_file)
+		or logit('err',
+			 "can't open $sys_git_projects_file for writing: $!");
+	}
+    }
+    if ($sys_git_cgitrepos_file) {
+	if (dry_run()) {
+	    open($cgit_fd, ">&STDOUT");
+	} else {
+	    open($cgit_fd, '>', $sys_git_cgitrepos_file)
+		or logit('err',
+			 "can't open $sys_git_cgitrepos_file for writing: $!");
+	}
+    }
+    if (defined($proj_fd) || defined($cgit_fd)) {
+	db_foreach(sub {
+	    my $ref = shift;
+	    $ref->{feature}{git}{dir_type} = $ref->{group_type_dir_type_git};
+	    delete $ref->{group_type_dir_type_git};
+	    $ref->{feature}{git}{dir} =
+		$ref->{dir_git} || $ref->{group_type_dir_git};
+	    delete $ref->{dir_git};
+	    delete $ref->{group_type_dir_git};
+
+	    my $repo_name;
+	    if ($ref->{master} eq 'Y') {
+		$repo_name = $ref->{repo_name};
+		delete $ref->{repo_name};
+	    } else {
+		$repo_name = $ref->{unix_group_name} . '/' . $ref->{repo_name};
+	    }
+	    
+	    
+	    my $obj = backend_feature('git', $ref);
+	    unless ($obj) {
+		local $Data::Dumper::Indent = 0;
+		local $Data::Dumper::Terse = 1;
+		local $Data::Dumper::Purity = 0;
+		logit('err', "can't create git object for ".Dumper([$ref]));
+		next;
+	    }
+	    my $dir_git = $obj->directory;
+	    
+	    my $owner = $ref->{owner};
+	    $owner =~ tr/ /+/;
+
+	    if (defined($proj_fd)) {
+		print $proj_fd "## $sys_git_projects_file ##\n" if dry_run();
+		print $proj_fd "$repo_name.git\t$owner\n";
+	    }
+	    if (defined($cgit_fd)) {
+		print $cgit_fd "## $sys_git_cgitrepos_file ##\n" if dry_run();
+		print $cgit_fd "repo.url=$repo_name.git\n";
+		print $cgit_fd "repo.path=$dir_git\n";
+		print $cgit_fd "repo.desc=$ref->{description}\n";
+		print $cgit_fd "repo.owner=$ref->{owner}\n";
+		print $cgit_fd "repo.readme=$dir_git/README.html\n";
+		print $cgit_fd "\n";
+	    }
+	},
+        q{
+SELECT g.unix_group_name as unix_group_name,
+       g.group_id as group_id,
+       g.type as type,
+       g.is_public as is_public,
+       g.gidNumber as gidNumber,
+       t.name as group_type_name,
+       g.dir_git as dir_git,
+       t.dir_git as group_type_dir_git,
+       t.dir_type_git as group_type_dir_type_git,
+       r.name as repo_name,
+       r.master as master,
+       r.owner as owner,
+       r.description as description,
+       t.dir_git as group_dir_git
+FROM git_repo r, groups g, group_type t
+WHERE g.group_id=r.group_id and g.type=t.type_id
+});
+	close $proj_fd if defined $proj_fd;
+	close $cgit_fd if defined $cgit_fd;
+    }
+}
+
+backend_done();
+
+=head1 NAME
+
+sv_groups - update system notion about Savane project groups
+
+=head1 SYNOPSIS
+
+B<sv_groups>
+[B<-F> I<FACILITY>]
+[B<-dn>]
+[B<-t> I<GROUPTYPE>]    
+[B<--cron>]
+[B<--debug>]
+[B<--dry-run>]
+[B<--facility=>I<FACILITY>]
+[B<--stderr>]
+[B<--syslog>]
+[B<--type=>I<GROUPTYPE>]
+    
+B<sv_groups> [B<-h>] [B<--help>] [B<--usage>]
+
+=head1 DESCRIPTION
+
+Creates and updates system files to reflect changes in the Savane projects.
+The following operations are performed:
+
+=over 4
+
+=item *
+
+For each new project registered, a corresponding UNIX group is created.
+No users are added to the group, this is responsibility of B<sv_users>,
+which normally is run after B<sv_groups>.
+
+=item *
+
+For any feature enabled in the project (both existing or just created),
+corresponding infrastructure is created in the file system.  What files
+are created depends on the feature in question: a directory or a Git, CVS,
+or other repository, etc.
+
+=item *    
+    
+Subordinate Git repositories are created, if necessary.
+
+=item *
+
+Updates in descriptions of Git repositories are replicated to the
+corresponding files, so that they are visible in tools like B<gitweb>
+and B<cgit>.
+
+=back    
+
+=head1 OPTIONS
+
+=over 8
+    
+=item B<--cron>
+
+Run only if the configuration variable B<$sys_cron_groups> is set to
+B<true>.  Implies B<--syslog>.
+    
+=item B<-d>, B<--debug>
+
+Increase debugging level.
+
+=item B<-n>, B<--dry-run>
+
+Do nothing, print everything.  Implies B<--debug>.
+
+=item B<-F>, B<--facility=>I<NAME>
+
+Log to the syslog facility I<NAME>.  If I<NAME> begins with a slash,
+it is understood as the name of the file where to direct the logging
+output.
+
+=item B<--syslog>
+
+Log to syslog.
+
+=item B<-t>, B<--type=>I<GROUPTYPE>
+
+=item B<--stderr>
+
+Log to standard error.  This is the default.
+    
+=item B<-h>
+
+Show a terse help summary and exit.
+
+=item B<--help>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 CONFIGURATION VARIABLES
+
+=over 4
+    
+=item B<sql.database>
+
+=item B<sql.user>
+
+=item B<sql.password>
+
+=item B<sql.host>
+
+=item B<sql.params>
+    
+Database credentials.    
+
+=item B<backend.sv_groups.cron>
+
+This variable is consulted if B<sv_groups> is run with the B<--cron>
+flag.  If it is B<1>, the program continues running.  If it is B<0>,
+B<sv_users> exits without doing anything.    
+    
+=item B<users.min_gid>
+
+Savane groups are assigned GIDs starting from that number.
+    
+=item B<path.userx_prefix>
+
+Directory where B<groupadd>(8) is located.
+    
+=item B<backend.sv_groups.project_list>
+
+Pathname of B<gitweb>-compatible F<.projects> file, where to store
+information about existent git repositories.    
+    
+=item B<backend.sv_groups.cgitrepos_file>
+
+Pathname of B<cgit> repository list file, where to store
+information about existent git repositories.
+    
+=back
+    
+=head1 EXIT CODES
+
+=over 8
+
+=item 0
+
+Successful termination.
+    
+=item 64
+
+Command line usage error.
+
+=item 78
+
+Error in configuration file.
+
+=back
+
+=head1 SEE ALSO
+
+B<sv_users>(1).
+    
+=head1 BUGS
+
+Debug output should be improved.  Success in I<dry-run> mode does not
+necessarily imply success in normal run: that depends on the availability
+of external tools and existence of directories.    
+    
+=head1 AUTHOR
+
+Sergey Poznyakoff <gray@gnu.org>
+    
+=cut    
+    
+
diff --git a/backend/accounts/sv_users_ldap b/backend/accounts/sv_users_ldap
new file mode 100755
index 0000000..890159d
--- /dev/null
+++ b/backend/accounts/sv_users_ldap
@@ -0,0 +1,148 @@
+#! /usr/bin/perl
+
+use strict;
+use Savane;
+use Savane::Conf;
+use Savane::Backend;
+use Savane::LDAP;
+use Savane::Homedir;
+use Text::Iconv;
+use POSIX qw(locale_h);
+
+setlocale(LC_ALL, "en_US.utf8");
+
+sub translit {
+    Text::Iconv->new("utf-8", "ascii//translit")->convert(@_);
+}
+
+sub array_eq {
+    my ($a, $b) = @_;
+    join('',sort @$a) eq join('',sort @$b)
+}
+
+my $sv_skel_dir = GetConf('backend.sv_users.skel_dir');
+my $sv_shell = GetConf('users.shell');
+
+backend_setup(descr => 'Synchronize Savane user database with LDAP',
+	      cron => GetConf('backend.sv_users.cron'));
+
+AcquireReplicationLock('groups-users.lock');
+
+my $ldap = new Savane::LDAP;
+my $sv_group = $ldap->getgrnam(GetConf('users.group'));
+
+# Collect user groups as per database
+my %db_user_group;
+foreach my $ent (GetDBHashArray("user_group,groups,user", 
+				"groups.group_id=user_group.group_id ".
+				"AND user.user_id=user_group.user_id ".
+				"AND groups.status='A' ".
+				"AND user_group.admin_flags<>'P'",
+				"user_name,unix_group_name")) {
+    push @{$db_user_group{$ent->{user_name}}}, $ent->{unix_group_name};
+}
+
+# Scan users
+foreach my $usr (GetDBHashArray("user",
+				@ARGV
+				  ? "status IN('A','D') AND user_name IN("
+				     . join(',', map { "'$_'" } @ARGV)
+				     . ")"
+				  : "status IN('A','D')",
+				"user_id,user_name,email,realname,authorized_keys,gpg_key,status,uidNumber")) {
+    debug("User=$usr->{user_name}, "
+	  . "real=$usr->{realname}, "
+	  . "email=$usr->{email}, "
+	  . "status=$usr->{status}");
+
+    my $pw = $ldap->getpwnam($usr->{user_name});
+
+    if ($pw && $pw->gid != $sv_group->gid) {
+	logit('info',
+	      "$usr->{user_name}: not in " . $sv_group->name . " group");
+    } elsif ($usr->{status} eq 'A') {
+	if ($pw) {
+	    my $gecos = translit($usr->{realname});
+	    $pw->gecos($gecos) if $gecos ne $pw->gecos;
+	    $pw->cn($usr->{realname}) if $usr->{realname} ne $pw->cn;
+
+	    my $db_keys = [split /###/, $usr->{authorized_keys}];
+	    my $pw_keys = $pw->sshpubkey // [];
+
+	    if (!array_eq($pw_keys, $db_keys)) {
+		$pw->sshpubkey($db_keys);
+	    }
+	} elsif ($db_user_group{$usr->{user_name}}) {
+	    # Create user
+	    my $h = new Savane::Homedir($usr->{user_name},
+					skel => $sv_skel_dir,
+					report => 1);
+	    # FIXME: create dir?
+	    $pw = new Savane::LDAP::pwent(name => $usr->{user_name},
+					  uid => $ldap->nextuid(GetConf('users.min_uid')),
+					  gid => $sv_group->gid,
+					  gecos => $usr->{realname},
+					  cn => $usr->{realname},
+					  dir => $h->dir,
+					  shell => $sv_shell,
+					  sshpubkey => [split /###/, $usr->{authorized_keys}]);
+	    logit('info', "adding LDAP user ".$pw->name);
+	    db_modify(qq{UPDATE user SET uidNumber=? WHERE user_id=?},
+		      $pw->uid, $usr->{user_id}) unless dry_run();
+	} else {
+	    next;
+	}
+	if (!dry_run) {
+	    $ldap->chpwent($pw)
+	}
+	my @pw_groups = $ldap->user_groups($pw->name);
+	my @db_groups = @{$db_user_group{$pw->name}//[]};
+	if (!array_eq([map { $_->name } @pw_groups], [@db_groups])) {
+	    #print STDERR "PW=".join(',',sort map { $_->name } @pw_groups)."\n";
+	    #print STDERR "DB=".join(',',sort @db_groups)."\n";
+	    foreach my $grp (@pw_groups) {
+		if (my @idx = grep { $db_groups[$_] eq $grp->name }
+		               (0 .. $#db_groups)) {
+		    foreach my $n (reverse @idx) {
+			splice @db_groups, $n, 1;
+		    }
+		} else {
+		    debug("group ".$grp->name.": remove ".$pw->name);
+		    $grp->mem_del($pw->name);
+		    $ldap->chgrent($grp) unless dry_run();
+		}
+	    }
+	}
+	foreach my $grp (map { $ldap->getgrnam($_) } @db_groups) {
+	    debug("group ".$grp->name.": add ".$pw->name);
+	    $grp->mem_add($pw->name);
+	    $ldap->chgrent($grp) unless dry_run();
+	}
+    } elsif ($pw) { # Status 'D'
+	logit('info', "deleting user $usr->{user_name}");
+	$ldap->delete($pw) unless dry_run();
+    }
+}
+
+logit('info','processing unassigned users');
+my $delete_unassigned = GetConf('backend.sv_users.remove_unassigned_users');
+my $deleted;
+
+foreach my $name ($ldap->user_names($sv_group->gid)) {
+    my ($res) = GetDBLists('user',"user_name=\"$name\"",'user_id');
+    unless ($res && GetDBLists('user_group',"user_id=$res->[0]", 'count(*)')) {
+	logit('info', "-- deleting user ".$name);
+	if ($delete_unassigned && !dry_run()) {
+	    $ldap->delete($ldap->getpwnam($name));
+	}
+	++$deleted;
+    }
+}
+
+if (!$delete_unassigned && $deleted) {
+    logit('info',
+	  'Set backend.sv_users.remove_unassigned_users = 1 to actually remove them');
+}
+
+
+