From c8b326ef70fbfc06483ae249219a1d4ab09c8bfe Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Sat, 2 Jan 2010 13:15:46 +0200 Subject: Implement distribution tarball checking. * src/config.c (event_args): New event "check-failure" (event_types): New event type ev_check_fail. (spool_kw,wydawca_kw): New keyword check-script. * src/directive.c (save_script) (stderr_redirector,run_check_script): New functions. (external_check): New function. (process_directives): Call external_check before actually moving the files. * src/gpg.c (homedir): Rename to temp_homedir, now global. * src/net.c (trim_crlf): Remove static qualifier. * src/triplet.c (hash_triplet_free): Free check_diag. (expand_triplet_full,expand_triplet_upload) (expand_triplet_sig,expand_triplet_directive): Rename to expand_triplet_ls_full,expand_triplet_ls_upload, expand_triplet_ls_sig,expand_triplet_ls_directive, correspondigly. (expand_check_diagn,expand_check_result) (expand_triplet_dist,expand_triplet_sig) (expand_triplet_dir): New functions. (triplet_meta): Renames: triplet:full => triplet:ls:full triplet:upload => triplet:ls:upload triplet:dist => triplet:ls:dist triplet:sig => triplet:ls:sig triplet:dir => triplet:ls:dir New keywords: triplet:dist, triplet:sig, triplet:dir, check:result, check:diagn. * src/wydawca.c (default_check_script): New global. (stat_name): New statistics counter "check failures". * src/wydawca.h (struct file_triplet): New members check_result, check_diag. (struct spool): New member check_script. (wydawca_stat): New value STAT_CHECK_FAIL. (notification_event): New value ev_check_fail. (default_check_script, temp_homedir): New externs. (concat_dir, copy_file, trim_crlf): New protos. * doc/wydawca.texi: Update. * configure.ac, NEWS: Version 2.0.90 --- NEWS | 28 +++++- configure.ac | 2 +- doc/wydawca.texi | 151 ++++++++++++++++++++++++++--- src/config.c | 11 ++- src/directive.c | 290 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/gpg.c | 20 ++-- src/net.c | 2 +- src/triplet.c | 100 +++++++++++++++---- src/wydawca.c | 3 +- src/wydawca.h | 10 ++ 10 files changed, 567 insertions(+), 50 deletions(-) diff --git a/NEWS b/NEWS index 73e5231..cad6ca1 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,36 @@ -Wydawca NEWS -- history of user-visible changes. 2009-12-22 +Wydawca NEWS -- history of user-visible changes. 2010-01-02 Copyright (C) 2007, 2008, 2009, 2010 Sergey Poznyakoff See the end of file for copying conditions. Please send Wydawca bug reports to . +Version 2.0.90 (Git) + +* Incompatible changes + +The following meta-variables are renamed: + + Old name New name + ---------------+--------------------- + triplet:full triplet:ls:full + triplet:upload triplet:ls:upload + triplet:dist triplet:ls:dist + triplet:sig triplet:ls:sig + triplet:dir triplet:ls:dir + +To update your configuration, use the following (extended) sed expression: + + s/\{(triplet):(full|upload|dist|sig|dir)\}/{\1:ls:\2}/g + +* Distribution verification. + +The new keyword `check-script' defines a shell script to +verify the submitted tarball. See the documentation, section +4.12 "Distribution Verification", for details. + + + Version 2.0, 2009-12-22 * Configuration file support rewritten from scratch. diff --git a/configure.ac b/configure.ac index 87b9d0d..437644d 100644 --- a/configure.ac +++ b/configure.ac @@ -15,7 +15,7 @@ # along with wydawca. If not, see . AC_PREREQ(2.63) -AC_INIT([wydawca], 2.0, [bug-wydawca@gnu.org.ua]) +AC_INIT([wydawca], 2.0.90, [bug-wydawca@gnu.org.ua]) AC_CONFIG_SRCDIR([src/wydawca.c]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_HEADER([config.h]) diff --git a/doc/wydawca.texi b/doc/wydawca.texi index 41307fb..88c2e6c 100644 --- a/doc/wydawca.texi +++ b/doc/wydawca.texi @@ -490,6 +490,7 @@ directives any time by running @command{wydawca --config-help}. * dictionaries:: * archivation:: * spool:: +* verification:: * statistics:: * notification:: @end menu @@ -1831,6 +1832,113 @@ spool ftp @{ @end group @end smallexample +@node verification +@section Distribution Verification +@cindex verification +@cindex distribution verification + +After the submission has been verified, @command{wydawca} may also run +an additional check to verify whether the main file (normally, +a tarball) is OK to be distributed. To set up such @dfn{distribution +verification}, add the following statement either in the global scope, +or within a @samp{spool} declaration: + +@deffn {Config} check-script @var{text} +@deffnx {Config:spool} check-script @var{text} +Define the distribution verification script. The @var{text} must +be a valid @file{sh} program. It is executed without arguments, +in a temporary directory which contains a copy of the main +distribution file. The script can refer to the following environment +variables: + +@defvr {Check Environment} WYDAWCA_SPOOL +Spool tag. +@end defvr + +@defvr {Check Environment} WYDAWCA_SOURCE +Spool source directory, as set by the @code{source} statement +(@pxref{spool,,tag}). +@end defvr + +@defvr {Check Environment} WYDAWCA_DEST +Spool destination directory (@pxref{spool,,destination}). +@end defvr + +@defvr {Check Environment} WYDAWCA_URL +Spool @acronym{URL} (@pxref{spool,,url}). +@end defvr + +@defvr {Check Environment} WYDAWCA_TRIPLET_BASE +Base name of the triplet. +@end defvr + +@defvr {Check Environment} WYDAWCA_DIST_FILE +File name of the main distribution file. +@end defvr + +Apart from these, the script inherits @command{wydawca} environment. + +The submission is accepted only if the script returns 0. Otherwise, +it is rejected and the @samp{check-failure} event (@pxref{event +notification}) is generated. + +In case of non-zero return, the script may return additional +diagnostics on the standard output. This diagnostics will be +available for use in notification messages via the @samp{check:diagn} +meta-variable. + +Additionally, the actual return code of the script, in decimal, is +available in the @samp{check:result} meta-variable. If the script +terminates on a signal, the value of this variable is +@samp{SIG+@var{n}}, where @var{n} is the signal number. +@end deffn + + If both global and spool @samp{check-script}s are defined, +@command{wydawca} executes both scripts as if they were connected +by a logical @samp{&&}, i.e. per-spool script is executed only if +the global one returned success (@samp{0}). The submission is accepted +only if both scripts returned @samp{0}. + + Since the script usually contains several lines, the +@samp{config-script} value is usually supplied using a here-document +construct (@pxref{here-document}). + + The following example illustrates the use of @samp{config-script} to +catch possible security holes in the distributed @file{Makefile.in} +files@footnote{See +@uref{http://article.gmane.org/gmane.comp.sysutils.autotools.announce/131}.} + +@smallexample + check-script <= 0) + { + trim_crlf (buf); + logmsg (LOG_NOTICE, "%s: %s", tag, buf); + } + _exit (0); + } + + close (p[0]); + return p[1]; +} + +static int +run_check_script (const char *script, struct file_triplet *trp, + const char *descr) +{ + static char *script_file; + pid_t pid; + int status; + int p[2]; + RETSIGTYPE (*oldsig)(); + FILE *fp; + char *buf; + size_t size, total; + struct obstack stk; + + if (debug_level > 1) + logmsg (LOG_DEBUG, _("prep script: %20.20s%s"), + script, strlen (script) > 20 ? "..." : ""); + script_file = save_script (script); + if (!script_file) + return 1; + if (debug_level > 1) + logmsg (LOG_DEBUG, _("script file: %s"), script_file); + + if (pipe (p)) + { + logmsg (LOG_CRIT, "pipe: %s", strerror (errno)); + return 1; + } + oldsig = signal (SIGCHLD, SIG_DFL); + pid = fork (); + if (pid == -1) + { + logmsg (LOG_CRIT, "fork: %s", strerror (errno)); + free (script_file); + close (p[0]); + close (p[1]); + signal (SIGCHLD, oldsig); + return 1; + } + if (pid == 0) + { + int i; + int efd; + char *argv[4]; + const struct spool *spool = trp->spool; + + signal (SIGHUP, SIG_DFL); + signal (SIGTERM, SIG_DFL); + signal (SIGQUIT, SIG_DFL); + signal (SIGINT, SIG_DFL); + signal (SIGCHLD, SIG_DFL); + signal (SIGALRM, SIG_DFL); + + efd = stderr_redirector (script_file); + if (efd == -1) + _exit (127); + + for (i = getdtablesize (); i > 0; i--) + { + if (i != p[1] && i != efd) + close (i); + } + + if (p[1] != 1 && dup2 (p[1], 1) != 1) + { + logmsg (LOG_CRIT, "cannot duplicate script's stdout: %s", + strerror (errno)); + _exit (127); + } + + if (efd != 2 && dup2 (efd, 2) != 2) + { + logmsg (LOG_CRIT, "cannot duplicate script's stderr: %s", + strerror (errno)); + _exit (127); + } + + setenv ("WYDAWCA_SPOOL", spool->tag, 1); + setenv ("WYDAWCA_SOURCE", spool->source_dir, 1); + setenv ("WYDAWCA_DEST", spool->dest_dir, 1); + setenv ("WYDAWCA_URL", spool->url, 1); + setenv ("WYDAWCA_TRIPLET_BASE", trp->name, 1); + setenv ("WYDAWCA_DIST_FILE", trp->file[file_dist].name, 1); + + chdir (temp_homedir); + + argv[0] = "-sh"; + argv[1] = script_file; + argv[2] = NULL; + + execv ("/bin/sh", argv); + _exit (127); + } + + /* Master */ + free (script_file); + close (p[1]); + fp = fdopen (p[0], "r"); + buf = NULL; + size = total = 0; + obstack_init (&stk); + if (debug_level > 2) + logmsg (LOG_DEBUG, _("reading script output...")); + while (getline (&buf, &size, fp) > 0) + { + size_t len = strlen (buf); + if (debug_level > 2) + logmsg (LOG_DEBUG, _("read: %s"), buf); + obstack_grow (&stk, buf, len); + total += size; + } + obstack_1grow (&stk, 0); + if (debug_level > 2) + logmsg (LOG_DEBUG, _("bytes read: %lu"), (unsigned long)total); + + fclose (fp); + + waitpid (pid, &status, 0); + signal (SIGCHLD, oldsig); + + if (total) + trp->check_diag = xstrdup (obstack_finish (&stk)); + obstack_free (&stk, NULL); + + trp->check_result = status; + if (WIFEXITED (status)) + { + status = WEXITSTATUS (status); + if (status) + { + logmsg (LOG_ERR, "%s for triplet %s, spool %s returned %d", + descr, trp->name, trp->spool->tag, status); + return 1; + } + else if (debug_level > 2) + logmsg (LOG_DEBUG, "%s for triplet %s, spool %s returned %d", + descr, trp->name, trp->spool->tag, status); + } + else if (WIFSIGNALED (status)) + { + int sig = WTERMSIG (status); + logmsg (LOG_NOTICE, + "%s for triplet %s, spool %s terminated on signal %d", + descr, trp->name, trp->spool->tag, sig); + return 1; + } + else + { + logmsg (LOG_NOTICE, + "%s for triplet %s, spool %s terminated with unhandled status", + descr, trp->name, trp->spool->tag); + return 1; + } + return 0; +} + +static int +external_check (struct file_triplet *trp) +{ + int rc; + const struct spool *spool = trp->spool; + char *file; + + if (!trp->file[file_dist].name) + return 0; + if (!spool->check_script && !default_check_script) + return 0; + + file = concat_dir (temp_homedir, trp->file[file_dist].name, NULL); + if (copy_file (trp->file[file_dist].name, file)) + { + free (file); + return 1; + } + + rc = 0; + if (spool->check_script) + rc |= run_check_script (spool->check_script, trp, + _("spool check script")); + + if (rc == 0 && default_check_script) + rc |= run_check_script (default_check_script, trp, + _("default check script")); + + free (file); + + if (rc) + { + UPDATE_STATS (STAT_CHECK_FAIL); + notify (spool->notification, trp, ev_check_fail); + } + + return rc; +} + /* Process the directives from TRP, using given SPOOL */ int process_directives (struct file_triplet *trp, const struct spool *spool) @@ -327,6 +614,7 @@ process_directives (struct file_triplet *trp, const struct spool *spool) break; case directory_dir: + /* FIXME: Alloc it in triplet */ relative_dir = safe_file_name_alloc (val); if (!relative_dir || relative_dir[0] == '/') { @@ -340,6 +628,8 @@ process_directives (struct file_triplet *trp, const struct spool *spool) rc = verify_detached_signature (trp, spool); if (rc == 0) { + if (external_check (trp)) + return 1; if (move_file (trp, spool, file_dist, relative_dir) || move_file (trp, spool, file_signature, relative_dir)) return 1; diff --git a/src/gpg.c b/src/gpg.c index f833142..cca5048 100644 --- a/src/gpg.c +++ b/src/gpg.c @@ -33,7 +33,7 @@ } \ while (0) -static char *homedir; +char *temp_homedir; static int rmdir_r (const char *name); @@ -122,29 +122,29 @@ static void remove_homedir () { if (debug_level > 1) - logmsg (LOG_DEBUG, _("removing GNUPG home directory: %s"), homedir); - if (rmdir_r (homedir)) - logmsg (LOG_CRIT, _("failed to remove GPG directory %s"), homedir); + logmsg (LOG_DEBUG, _("removing GNUPG home directory: %s"), temp_homedir); + if (rmdir_r (temp_homedir)) + logmsg (LOG_CRIT, _("failed to remove GPG directory %s"), temp_homedir); } /* Create a temporary GPG home directory */ static int create_gpg_homedir () { - if (homedir) + if (temp_homedir) return 0; - homedir = xstrdup ("/tmp/wydawca-XXXXXX"); - if (!mkdtemp (homedir)) + temp_homedir = xstrdup ("/tmp/wydawca-XXXXXX"); + if (!mkdtemp (temp_homedir)) { logmsg (LOG_CRIT, _("cannot create GPG home directory (%s): %s"), - homedir, strerror (errno)); + temp_homedir, strerror (errno)); return 1; } atexit (remove_homedir); if (debug_level > 1) - logmsg (LOG_DEBUG, _("GNUPG home directory: %s"), homedir); - setenv ("GNUPGHOME", homedir, 1); + logmsg (LOG_DEBUG, _("GNUPG home directory: %s"), temp_homedir); + setenv ("GNUPGHOME", temp_homedir, 1); return 0; } diff --git a/src/net.c b/src/net.c index 2379cb3..e6bb4ab 100644 --- a/src/net.c +++ b/src/net.c @@ -82,7 +82,7 @@ open_listener () return fd; } -static void +void trim_crlf (char *s) { size_t len = strlen (s); diff --git a/src/triplet.c b/src/triplet.c index dbba486..f18f573 100644 --- a/src/triplet.c +++ b/src/triplet.c @@ -52,6 +52,7 @@ hash_triplet_free (void *data) free (tp->directive); free (tp->blurb); free (tp->tmp); + free (tp->check_diag); /* FIXME: free uploader list */ free (tp); @@ -368,7 +369,7 @@ format_file_data (struct file_triplet *trp, enum file_type type, char **pret) } static const char * -expand_triplet_full (struct metadef *def, void *data) +expand_triplet_ls_full (struct metadef *def, void *data) { struct file_triplet *trp = data; char *buf[FILE_TYPE_COUNT] = { NULL, NULL, NULL }; @@ -406,7 +407,7 @@ expand_triplet_full (struct metadef *def, void *data) } static const char * -expand_triplet_upload (struct metadef *def, void *data) +expand_triplet_ls_upload (struct metadef *def, void *data) { struct file_triplet *trp = data; char *buf[2] = { NULL, NULL }; @@ -439,6 +440,27 @@ expand_triplet_upload (struct metadef *def, void *data) static const char * expand_triplet_dist (struct metadef *def, void *data) +{ + struct file_triplet *trp = data; + return trp->file[file_dist].name; +} + +static const char * +expand_triplet_sig (struct metadef *def, void *data) +{ + struct file_triplet *trp = data; + return trp->file[file_signature].name; +} + +static const char * +expand_triplet_dir (struct metadef *def, void *data) +{ + struct file_triplet *trp = data; + return trp->file[file_directive].name; +} + +static const char * +expand_triplet_ls_dist (struct metadef *def, void *data) { struct file_triplet *trp = data; format_file_data (trp, file_dist, &def->storage); @@ -447,7 +469,7 @@ expand_triplet_dist (struct metadef *def, void *data) } static const char * -expand_triplet_sig (struct metadef *def, void *data) +expand_triplet_ls_sig (struct metadef *def, void *data) { struct file_triplet *trp = data; format_file_data (trp, file_signature, &def->storage); @@ -456,7 +478,7 @@ expand_triplet_sig (struct metadef *def, void *data) } static const char * -expand_triplet_directive (struct metadef *def, void *data) +expand_triplet_ls_directive (struct metadef *def, void *data) { struct file_triplet *trp = data; format_file_data (trp, file_directive, &def->storage); @@ -509,6 +531,37 @@ expand_comment (struct metadef *def, void *data) return def->value; } +static const char * +expand_check_diagn (struct metadef *def, void *data) +{ + struct file_triplet *trp = data; + return trp->check_diag; +} + +static const char * +expand_check_result (struct metadef *def, void *data) +{ + struct file_triplet *trp = data; + int status = trp->check_result; + char sbuf[INT_BUFSIZE_BOUND (uintmax_t)]; + + if (status == 0) + return def->value = "0"; + else if (WIFEXITED (status)) + { + char *p = umaxtostr (WEXITSTATUS (status), sbuf); + def->storage = xstrdup (p); + } + else if (WIFSIGNALED (status)) + { + char *p = umaxtostr (WTERMSIG (status), sbuf); + asprintf (&def->storage, "SIG+%s", p); + } + else + def->storage = "[unrecognized return code]"; + return def->value = def->storage; +} + #define DECL_EXPAND_TIMER(what) \ static const char * \ __cat2__(expand_timer_,what) (struct metadef *def, void *data) \ @@ -555,23 +608,28 @@ triplet_expand_dictionary_query (struct dictionary *dict, void *handle, } struct metadef triplet_meta[] = { - { "project", NULL, expand_project_base, NULL }, - { "url", NULL, expand_url, NULL }, - { "spool", NULL, expand_tag, NULL }, - { "dir", NULL, expand_relative_dir, NULL }, - { "dest-dir", NULL, expand_dest_dir, NULL }, - { "source-dir", NULL, expand_source_dir, NULL }, - { "triplet:full", NULL, expand_triplet_full, NULL }, - { "triplet:upload", NULL, expand_triplet_upload, NULL }, - { "triplet:dist", NULL, expand_triplet_dist, NULL }, - { "triplet:sig", NULL, expand_triplet_sig, NULL }, - { "triplet:dir", NULL, expand_triplet_directive, NULL }, - { "user", NULL, expand_user_name, NULL }, - { "user:name", NULL, expand_user_name, NULL }, - { "user:real-name", NULL, expand_user_real_name, NULL }, - { "user:email", NULL, expand_user_email, NULL }, - { "report", NULL, expand_report, NULL }, - { "comment", NULL, expand_comment, NULL }, + { "project", NULL, expand_project_base, NULL }, + { "url", NULL, expand_url, NULL }, + { "spool", NULL, expand_tag, NULL }, + { "dir", NULL, expand_relative_dir, NULL }, + { "dest-dir", NULL, expand_dest_dir, NULL }, + { "source-dir", NULL, expand_source_dir, NULL }, + { "triplet:dist", NULL, expand_triplet_dist, NULL }, + { "triplet:sig", NULL, expand_triplet_sig, NULL }, + { "triplet:dir", NULL, expand_triplet_dir, NULL }, + { "triplet:ls:full", NULL, expand_triplet_ls_full, NULL }, + { "triplet:ls:upload", NULL, expand_triplet_ls_upload, NULL }, + { "triplet:ls:dist", NULL, expand_triplet_ls_dist, NULL }, + { "triplet:ls:sig", NULL, expand_triplet_ls_sig, NULL }, + { "triplet:ls:dir", NULL, expand_triplet_ls_directive, NULL }, + { "check:result", NULL, expand_check_result, NULL }, + { "check:diagn", NULL, expand_check_diagn, NULL }, + { "user", NULL, expand_user_name, NULL }, + { "user:name", NULL, expand_user_name, NULL }, + { "user:real-name", NULL, expand_user_real_name, NULL }, + { "user:email", NULL, expand_user_email, NULL }, + { "report", NULL, expand_report, NULL }, + { "comment", NULL, expand_comment, NULL }, DECL_FULL_TIMER(wydawca), DECL_FULL_TIMER(triplet), DECL_FULL_TIMER(spool), diff --git a/src/wydawca.c b/src/wydawca.c index 5dc73e7..fc1dea2 100644 --- a/src/wydawca.c +++ b/src/wydawca.c @@ -46,7 +46,7 @@ int single_process; time_t wakeup_interval; gl_list_t all_spool_aliases; char *wydawca_gpg_homedir; - +char *default_check_script; struct grecs_sockaddr listen_sockaddr; unsigned wydawca_stat[MAX_STAT]; @@ -142,6 +142,7 @@ static char *stat_name[MAX_STAT] = { N_("files archived"), N_("symlinks created"), N_("symlinks removed"), + N_("check failures"), }; static char *stat_kwname[MAX_STAT] = { diff --git a/src/wydawca.h b/src/wydawca.h index b2e79d4..af24e47 100644 --- a/src/wydawca.h +++ b/src/wydawca.h @@ -184,6 +184,8 @@ struct file_triplet struct uploader_info *uploader; /* Special data for template formatting */ char *project; /* Triplet project name (if known) */ + int check_result; /* Result of external check */ + char *check_diag; /* External check diagnostics */ }; /* Macros to access owner UID and GID. */ @@ -233,6 +235,7 @@ struct spool struct dictionary *dictionary[dictionary_count]; struct archive_descr archive; /* Archivation data */ struct notification *notification; + char *check_script; }; @@ -251,6 +254,7 @@ enum wydawca_stat STAT_ARCHIVES, STAT_SYMLINKS, STAT_RMSYMLINKS, + STAT_CHECK_FAIL, MAX_STAT }; @@ -265,6 +269,7 @@ enum notification_event ev_bad_ownership, ev_bad_directive_signature, ev_bad_detached_signature, + ev_check_fail, MAX_EVENT }; @@ -347,6 +352,8 @@ extern struct grecs_sockaddr listen_sockaddr; extern gl_list_t all_spool_aliases; extern char *wydawca_gpg_homedir; +extern char *default_check_script; +extern char *temp_homedir; #define UPDATE_STATS(what) \ do \ @@ -477,6 +484,8 @@ rmsymlink_file (struct file_triplet *trp, const struct spool *spool, /* diskio.c */ +char *concat_dir (const char *base, const char *name, size_t *pbaselen); +int copy_file (const char *file, const char *dst_file); int dir_test_url (mu_url_t url, grecs_locus_t *locus); int dir_move_file (struct file_triplet *trp, const struct spool *spool, @@ -536,6 +545,7 @@ void remove_pidfile (void); /* net.c */ void wydawca_listener (void); +void trim_crlf (char *s); #define LOCK_OK 0 -- cgit v1.2.1