diff options
53 files changed, 1343 insertions, 1014 deletions
@@ -1,4 +1,4 @@ -Wydawca NEWS -- history of user-visible changes. 2020-04-20 +Wydawca NEWS -- history of user-visible changes. 2020-04-23 See the end of file for copying conditions. Please send Wydawca bug reports to <bug-wydawca@gnu.org.ua>. @@ -46,12 +46,23 @@ spool simultaneously lock files were used. This version does not allow for such usage. Only one copy of wydawca can be running. Therefore spool locking has been discontinued. +* Command line usage + +** The --single-process option removed + +** The -d (--debug) option requires argument + +The argument is a decimal number specifying the desired debug level. +Thus, instead of -ddd, use -d3 (or --debug=3). + * Configuration changes ** The wakeup-interval statement removed ** The locking statement removed +** The single-process statement removed + ** max-connections statement This new statement configures the maximum number of upload diff --git a/configure.ac b/configure.ac index 0e1366b..865d206 100644 --- a/configure.ac +++ b/configure.ac @@ -110,7 +110,7 @@ AH_BOTTOM( #endif ]) LDADD=$saved_LDADD - + # ********************** # TCP wrappers # ********************** @@ -176,10 +176,24 @@ AM_MISSING_PROG([AUTOM4TE], [autom4te]) # Initialize documentation helpers. IMPRIMATUR_INIT(, [frenchspacing]) -AC_PATH_PROG(GPG, gpg, '') -if test -n "$GPG"; then - AC_SUBST(WYDAWCA_GPGFILES, '$(GPGFILES)') -fi +# ********************** +# GPG v2 (for the testsuite) +# ********************** +AC_ARG_VAR([GPG],[Name of the gpg version 2 binary]) +AC_MSG_CHECKING([for the GPG v2 binary]) +uGPG=$GPG +unset GPG +for prog in $uGPG gpg2 gpg +do + ver=$($prog --version 2>/dev/null | head -n 1) + case $ver in + "gpg (GnuPG) 2."*) + GPG=$prog + break;; + esac +done +AC_MSG_RESULT([${GPG:-none}${GPG:+, $ver}]) +AM_CONDITIONAL([COND_GPG2],[test -n "$GPG"]) AC_CONFIG_FILES([Makefile doc/Makefile diff --git a/doc/wydawca.texi b/doc/wydawca.texi index 64a6e36..b05e8b0 100644 --- a/doc/wydawca.texi +++ b/doc/wydawca.texi @@ -444,16 +444,13 @@ $ wydawca --spool=ftp --spool=test --source=/home/ftp/test-upload @anchor{debug} @xopindex{debug, described} @sopindex{d, described} - The @option{--debug} (@option{-d}) option tells the program to increase its -debugging level by 1. The @dfn{debugging level} determines amount -of information the program reports when it runs. Default level is 0, -which means that only errors and other critical conditions are -reported. Raising it may be necessary when debugging new configurations. Each -@option{-d} option raises the level by one, so you can say -@command{wydawca -dd} to obtain level 2, for example. The maximum -debugging level (currently it is 4) prints an impractically big -amount of information, and is useful mainly for @command{wydawca} -developers. + The @option{--debug} (@option{-d}) option tells the program to set its +debugging level to the given integer value. @dfn{Debugging level} +determines the amount of information the program reports when it +runs. Default level is 0, which means that only errors and other +critical conditions are reported. Raising it may be necessary when +debugging new configurations. @samp{Wydawca} version @value{version} +implements 4 distinct debuggin levels. @anchor{dry-run} @xopindex{dry-run, described} @@ -463,9 +460,11 @@ developers. modifications to the disk contents, and to print a verbose description of any actions it would have taken. It sets the debugging level to 1 and directs the diagnostics output to the standard error, as if -@option{--debug --stderr} options were given. You can raise -debugging level further by supplying additional @option{--debug} -options. The @option{--dry-run} option is useful when testing new +@option{--debug=1 --stderr} options were given. You can further +control the debugging level by supplying additional @option{--debug} +options @emph{after} the @option{--dry-run} option. + +The @option{--dry-run} option is useful when testing new configurations, for example: @smallexample @@ -498,10 +497,11 @@ reports them on its error output and exits with a non-zero status. other inconsistencies. If no errors were detected, the program exits with code 0. Otherwise, the exit code is 78. - Using this option together with @option{-d} (@option{--debug}), + Using this option together with @option{-d1} (@option{--debug=1}), causes @command{wydawca} to produce a dump of the configuration parse -tree. Using the @option{-d} option twice prefixes each statement in -the dump with the file location where it appeared. +tree. Setting a higher debugging level (e.g. @option{-d2} option) will +additionally prefix each statement in the dump with the file location +where it appeared. @sopindex{E, introduced} @xopindex{no-preprocessor, introduced} @@ -980,15 +980,6 @@ If @var{bool} is @samp{yes}, run in foreground. @xref{invocation, foreground}. @end deffn -@deffn {Config} single-process bool -Configure single process mode. Normally @command{wydawca} -spawns subprocesses for handling incoming connections and spool jobs. -This is disabled if @var{bool} is @samp{yes} (a so-called -@dfn{single-processs mode}). This mode is designed for debugging -purposes. Do not use it in production environments, because it -severely impairs performance. -@end deffn - @deffn {Config} umask value Set the default umask. The @var{value} argument must be an octal number. @end deffn @@ -3328,10 +3319,6 @@ daemon @var{arg:@i{boolean}}; # @xref{general, foreground}. foreground @var{arg:@i{boolean}}; -# @r{Do not spawn subprocesses.} -# @xref{general, single-process}. -single-process @var{arg:@i{boolean}}; - # @r{Set pid file name.} # @xref{general, pidfile}. pidfile @var{file:@i{string}}; @@ -3655,9 +3642,9 @@ Run in daemon mode. @xref{starting,, daemon}. @opsummary{debug} @sopindex{d, summary} -@item --debug -@itemx -d -Increase debugging level by 1. +@item --debug=@var{n} +@itemx -d @var{n} +Set the debugging level to @var{n}. @xref{debug, The @option{--debug} option}. @@ -3683,7 +3670,7 @@ Dump lexical analyzer traces. This is useful for debugging @item --dry-run @itemx -n @dfn{Dry-run mode}: do nothing, print almost everything. This option -implies @option{--debug --stderr}. +implies @option{--debug=1 --stderr}. @xref{dry-run, The dry-run mode}. @@ -3741,11 +3728,6 @@ Disable preprocessor. @pxref{Preprocessor}. @item --preprocessor=@var{command} Use @var{command} instead of the default preprocessor. @pxref{Preprocessor}. -@opsummary{single-process} -@item --single-process -Serialize job invocations by not forking subprocesses for each job. -@emph{Do not use this option in production environment}. - @opsummary{source} @sopindex{s, summary} @item --source=@var{name} diff --git a/src/cmdline.opt b/src/cmdline.opt index fae3cb1..a23335d 100644 --- a/src/cmdline.opt +++ b/src/cmdline.opt @@ -96,12 +96,6 @@ BEGIN foreground_option = 1; END -OPTION(single-process,,, - [<single process mode>]) -BEGIN - single_process_option = 1; -END - OPTION(config-file,c,FILE, [<use FILE instead of the default configuration>]) BEGIN @@ -180,10 +174,10 @@ END GROUP(Debugging) -OPTION(debug,d,, - [<increase debugging level>]) +OPTION(debug,d,[<LEVEL>], + [<set debugging level>]) BEGIN - wy_debug_level++; + wy_debug_level = atoi(optarg); END OPTION(dump-grammar-trace,,, diff --git a/src/config.c b/src/config.c index 9d78ed3..6b99801 100644 --- a/src/config.c +++ b/src/config.c @@ -1390,9 +1390,6 @@ static struct grecs_keyword wydawca_kw[] = { { "foreground", NULL, N_("Start in foreground even in daemon mode"), grecs_type_bool, GRECS_DFLT, &foreground }, - { "single-process", NULL, - N_("Do not spawn subprocesses"), - grecs_type_bool, GRECS_DFLT, &single_process }, { "pidfile", N_("file"), N_("Set pid file name"), grecs_type_string, GRECS_CONST, &pidfile}, @@ -1727,7 +1724,7 @@ config_finish(struct grecs_node *tree) if (file_sweep_time <= 0) { file_sweep_time = DEFAULT_FILE_SWEEP_TIME; wy_log(LOG_NOTICE, - _("file-sweep-time too low; reverting to the default %ds"), - file_sweep_time); + _("file-sweep-time too low; reverting to the default %lus"), + (unsigned long)file_sweep_time); } } diff --git a/src/triplet.c b/src/triplet.c index bde3c14..980c906 100644 --- a/src/triplet.c +++ b/src/triplet.c @@ -355,9 +355,10 @@ void triplet_enqueue(struct wy_triplet *trp) { pthread_t tid; - struct spool *spool = trp->spool; + struct spool *spool; if (!trp) return; + spool = trp->spool; timer_start(WY_TIMER_SPOOL); timer_start(spool->timer_id); if (spool_open_dictionaries(spool) == 0) { @@ -511,12 +512,19 @@ wy_thr_cleaner(void *ptr) void wy_triplet_wait(void) { + size_t n; + triplet_list_lock(&triplet_running_list); while (triplet_running_list.head) { pthread_cond_wait(&triplet_running_list.cond, &triplet_running_list.mutex); } triplet_list_unlock(&triplet_running_list); + + pthread_mutex_lock(&triplet_table_mutex); + n = grecs_symtab_count(triplet_table); + pthread_mutex_unlock(&triplet_table_mutex); + *wy_get_stat_slot(WY_STAT_INCOMPLETE_TRIPLETS) += n; } static pthread_key_t key; diff --git a/src/wydawca.c b/src/wydawca.c index 01ef56c..5771feb 100644 --- a/src/wydawca.c +++ b/src/wydawca.c @@ -44,10 +44,8 @@ int wy_mode_option = -1; int preprocess_only = 0; int foreground_option = -1; -int single_process_option = -1; int foreground; -int single_process; time_t wakeup_interval; struct grecs_list *all_spool_aliases; char *wy_gpg_homedir; @@ -417,8 +415,6 @@ main(int argc, char **argv) wy_mode = wy_mode_option; if (foreground_option >= 0) foreground = foreground_option; - if (single_process_option >= 0) - single_process = single_process_option; if (wy_log_to_stderr == -1) { switch (wy_mode) { diff --git a/src/wydawca.h b/src/wydawca.h index 90a886e..b3ece49 100644 --- a/src/wydawca.h +++ b/src/wydawca.h @@ -330,7 +330,6 @@ enum { extern int wy_mode; extern int foreground; -extern int single_process; extern struct grecs_sockaddr listen_sockaddr; extern size_t max_connections; extern time_t idle_timeout; diff --git a/tests/.gitignore b/tests/.gitignore index e5a908e..6a471f2 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,3 +10,4 @@ cwdrepl wyasync wyclt wyinit +chargen diff --git a/tests/Makefile.am b/tests/Makefile.am index fb971d5..8374916 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Wydawca. If not, see <http://www.gnu.org/licenses/>. -EXTRA_DIST = $(TESTSUITE_AT) testsuite package.m4 dist +EXTRA_DIST = $(TESTSUITE_AT) testsuite package.m4 DISTCLEANFILES = atconfig $(check_SCRIPTS) MAINTAINERCLEANFILES = Makefile.in $(TESTSUITE) SUBDIRS = etc @@ -40,31 +40,33 @@ $(srcdir)/package.m4: $(top_srcdir)/configure.ac ## ------------ ## TESTSUITE_AT = \ + testsuite.at\ backup00.at\ backup01.at\ backup02.at\ backup03.at\ + dry_run00.at\ + dry_run01.at\ + dry_run02.at\ + upload00.at\ + upload01.at\ + upload02.at\ + symlink00.at\ + symlink01.at\ + rmsymlink00.at\ + mailnotify.at\ + mailstats.at\ + check-ok.at\ check-fail.at\ check-notify.at\ - check-ok.at\ inotify-ok.at\ - inotify-rmsymlink.at\ inotify-symlink.at\ + inotify-rmsymlink.at\ inotify-unatt00.at\ inotify-unatt01.at\ - notify-upl.at\ - mailstats.at\ - rmsymlink00.at\ - symlink00.at\ - symlink01.at\ - testsuite.at\ - upload.at\ - upload-dry.at\ - upl11.at\ - upl12f.at\ - upl12t.at\ unp00.at\ unp01.at\ + unp02.at\ unp-idle.at\ version.at @@ -89,7 +91,7 @@ check-local: atconfig atlocal $(TESTSUITE) #installcheck-local: # $(SHELL) $(TESTSUITE) AUTOTEST_PATH=$(exec_prefix)/bin -check_PROGRAMS = bkupname cwdrepl wyinit wyasync wyclt +check_PROGRAMS = bkupname cwdrepl wyinit wyasync wyclt chargen AM_CPPFLAGS = \ @GRECS_INCLUDES@ \ -I$(top_srcdir)/include\ diff --git a/tests/atlocal.in b/tests/atlocal.in index 6102995..0d7c732 100644 --- a/tests/atlocal.in +++ b/tests/atlocal.in @@ -8,66 +8,130 @@ XFAILFILE=$abs_builddir/.badversion trap "test -r $XFAILFILE && cat $XFAILFILE; exit $?" 1 2 13 15 -WY_CONFSRC=@abs_top_srcdir@/tests/etc -WY_DISTDIR=@abs_top_srcdir@/tests/dist -WY_SRC=@abs_top_builddir@/tests/source -WY_DST=dest WY_TESTDIR=@abs_top_builddir@/tests +WY_CONFSRC=@abs_top_srcdir@/tests/etc +WY_CONFDST=$WY_TESTDIR/etc +WY_PROJECTDB=$WY_CONFSRC/project WY_MAILUTILS=@WY_MAILUTILS@ WY_INOTIFY=@WY_INOTIFY@ -MKDIR_P="@MKDIR_P@" +WY_MODDIR=@abs_top_builddir@/modules/mailutils -if test -w /; then - WY_FORCE=--force -else - WY_FORCE= -fi +GPG="@GPG@ --quiet --no-permission-warning --batch " +GNUPGHOME=$WY_CONFDST/gnupg +export GNUPGHOME ulimit -c unlimited -wydawca_expandmeta() { - sed -e "s/@USER@/$USER/g" \ - -e "s,@CWD@,$PWD,g" \ - -e "s,@WY_SRC@,$WY_SRC,g" \ - -e "s,@WY_DST@,$WY_DST,g" \ - -e "s,@WY_CONFSRC@,$WY_CONFSRC,g" \ - -e "s,@WY_TESTDIR@,$WY_TESTDIR,g" \ - -e "s,@WY_MODDIR@,@abs_top_builddir@/modules/mailutils,g" $1 > $2 -} +# ##################################### +# Utility functions +# ##################################### -wydawca_config() { - wydawca_expandmeta $WY_CONFSRC/${1}in $1 - wyinit $1 +# wy_project_lookup PROJECT WHAT +wy_project_lookup() { + test $# -eq 2 || echo >&2 "wy_project_lookup: bad number of arguments" + local n + case $2 in + PROJECT_NAME) n=1;; + REAL_NAME) n=2;; + EMAIL) n=3;; + COMMENT) n=4;; + esac + grep "^$1:" $WY_PROJECTDB | cut -d: -f$n } + +# wy_create_directive [-v X.Y] [-u USER] FILENAME PROJECT [DIRECTIVE: VALUE ...] +wy_create_directive() { + local version user + while [ $# -ne 0 ] + do + case $1 in + -v) version=$2 + shift 2;; + -u) user=$2 + shift 2;; + --) shift + break;; + *) break + esac + done -wydawca_upload() { - dir=source/$1 - shift - file=$1 - shift - cp "$WY_DISTDIR/$file" "$dir" - cp "$WY_DISTDIR/$file.sig" "$dir" - if test -n "$1"; then - cp "$WY_DISTDIR/$1" "$dir/$file.directive.asc" - else - cp "$WY_DISTDIR/$file.directive.asc" "$dir" + : ${version:=1.2} + + local filename="$1.directive" project=$2 + shift 2 + if [ $(( $# % 2 )) -ne 0 ]; then + echo >&2 "wy_create_directive: odd number of arguments" + exit 1 fi + (echo "version: $version" + echo "directory: $project" + while [ $# -gt 0 ] + do + echo "$1 $2" + shift 2 + done) > $filename + rm -f $filename.asc + : ${user:=$(wy_project_lookup $project REAL_NAME)} + rm -f $filename.asc + $GPG --clearsign -u "$user" -o $filename.asc $filename + rm $filename } -wydawca_cmp() { - p=$1 - shift - for file in $@ +# wy_genfile [-u USER] [-l SIZE] PROJECT NAME +wy_genfile() { + local length user + while [ $# -ne 0 ] do - cmp "$WY_DISTDIR/$file" "$WY_DST/$p/$file" || exit 1 + case $1 in + -l) length="-l $2" + shift 2;; + -u) user=$2 + shift 2;; + --) shift + break;; + *) break + esac done + + local project=$1 filename=$2 + chargen $length > $filename + + : ${user:=$(wy_project_lookup $project REAL_NAME)} + rm -f $filename.sig + $GPG -ba -o $filename.sig $filename } -wydawca_cmparc() { - p=$1 - shift - for file in $@ +# wy_create_upload [-u USER] [-l SIZE] PROJECT NAME [DIRECTIVE: VALUE...] +wy_create_upload() { + local length_option user_option + while [ $# -ne 0 ] do - cmp "$WY_DISTDIR/$file" "$WY_DST/$p/archive/$file" || exit 1 + case $1 in + -l) + length_option="$1 $2" + shift 2;; + -u) + user_option="$1 $2" + shift 2;; + --) shift + break;; + *) break + esac done + local project=$1 filename=$2 + shift 2 + wy_genfile $length_option $user_option $project $filename + wy_create_directive $user_option $filename $project\ + filename: $(basename $filename)\ + "$@" +} + +wy_config_head() { + cat <<EOF +pidfile $PWD/wydawca.pid; +statistics all; +umask 022; +create-directories yes; +#include "$WY_CONFDST/database.cf" +EOF } diff --git a/tests/chargen.c b/tests/chargen.c new file mode 100644 index 0000000..13ef8f5 --- /dev/null +++ b/tests/chargen.c @@ -0,0 +1,79 @@ +/* + NAME + chargen - generate a stream of characters + + SYNOPSIS + chargen [-c C] [-l LEN] [-p] [-s N] + + DESCRIPTION + Produces on standard output a stream of characters. The stream consists + of all 256 characters repeated cyclically until total number of characters + reaches 4096 (or LEN). + + OPTIONS + -c C Start from ASCII character C + + -l LEN + Stop when LEN characters have been generated. + + -p Produce only printable characters + + -s N Start from character with ordinal number N +*/ +#include <stdio.h> +#include <stdlib.h> +#include <limits.h> +#include <unistd.h> +#include <errno.h> +#include <ctype.h> + +int +main(int argc, char **argv) +{ + int c; + int start = 0; + int mod = UCHAR_MAX; + size_t i, len = 4096; + int printable = 0; + char *p; + + while ((c = getopt(argc, argv, "cl:ps:")) != EOF) { + switch (c) { + case 'c': + start = optarg[0]; + break; + case 'p': + printable = 1; + break; + case 's': + start = atoi(optarg) % UCHAR_MAX; + break; + case 'l': + errno = 0; + len = strtoul(optarg, &p, 10); + if (*p) { + fprintf(stderr, "bad length (near %s)", p); + exit(1); + } else if (errno) { + perror("bad length"); + exit(1); + } + break; + default: + exit(2); + } + } + + if (printable && !isprint(start)) + do { + start = (start + 1) % mod; + } while (printable && !isprint(start)); + + for (i = 0; i < len; i++) { + putchar(start); + do { + start = (start + 1) % mod; + } while (printable && !isprint(start)); + } + return 0; +} diff --git a/tests/check-fail.at b/tests/check-fail.at index 268ca47..4c4b1c9 100644 --- a/tests/check-fail.at +++ b/tests/check-fail.at @@ -16,15 +16,28 @@ AT_SETUP([Verification test failure]) AT_KEYWORDS([uploads check-test check-fail]) - -AT_DATA([experr], +AT_CHECK_UPLOAD([ +spool test { + url ftp://wydawca.test; + source "$PWD/upload"; + destination "$PWD/dest"; + check-script "exit 1"; +} +], +[wy_create_upload conversion upload/conversion-1.1.tar +cp -a upload orig +set -e +wydawca --no-preproc -c wydawca.cf 2>err +cat err | cwdrepl >&2 +], +[0], +[], [wydawca: [[NOTICE]] AT_PACKAGE_TARNAME (AT_PACKAGE_NAME AT_PACKAGE_VERSION) started -wydawca: [[NOTICE]] file.directive.asc: VERSION: 1.1 -wydawca: [[NOTICE]] file.directive.asc: COMMENT: Gnupload for Wydawca testsuite -wydawca: [[ERR]] spool check script for file@ckfail returned 1 -wydawca: [[NOTICE]] removing ./source/fail/file -wydawca: [[NOTICE]] removing ./source/fail/file.sig -wydawca: [[NOTICE]] removing ./source/fail/file.directive.asc +wydawca: [[NOTICE]] conversion-1.1.tar.directive.asc: VERSION: 1.2 +wydawca: [[ERR]] spool check script for conversion-1.1.tar@test returned 1 +wydawca: [[NOTICE]] removing ./upload/conversion-1.1.tar +wydawca: [[NOTICE]] removing ./upload/conversion-1.1.tar.sig +wydawca: [[NOTICE]] removing ./upload/conversion-1.1.tar.directive.asc wydawca: [[INFO]] errors: 1 wydawca: [[INFO]] warnings: 0 wydawca: [[INFO]] bad signatures: 0 @@ -41,16 +54,6 @@ wydawca: [[INFO]] symlinks removed: 0 wydawca: [[INFO]] check failures: 1 wydawca: [[NOTICE]] AT_PACKAGE_TARNAME (AT_PACKAGE_NAME AT_PACKAGE_VERSION) finished ]) +AT_CLEANUP -AT_CHECK([ -wydawca_config wydawca.cf -wydawca_upload fail file - -wydawca $WY_FORCE --no-preproc -c wydawca.cf 2>err -cat err | cwdrepl >&2 -], -[0], -[], -[experr]) -AT_CLEANUP diff --git a/tests/check-notify.at b/tests/check-notify.at index 298ebd0..63a00a4 100644 --- a/tests/check-notify.at +++ b/tests/check-notify.at @@ -17,14 +17,67 @@ AT_SETUP([Verification test failure notification]) AT_KEYWORDS([uploads check-test check-fail check-fail-notify notify]) -AT_DATA([experr], +AT_CHECK_UPLOAD([ +module-prepend-load-path "$WY_MODDIR"; +module mailutils mod_mailutils.la; +module-init mailutils { + admin-address "root@localhost"; + from-address "wydawca-noreply@localhost"; + mailer "| $WY_TESTDIR/nullmail -o $PWD/mail.out -F \${sender} \${rcpt}"; +} + +#include "$WY_CONFSRC/notify.cf" + +spool test { + url ftp://wydawca.test; + source "$PWD/upload"; + destination "$PWD/dest"; + check-script <<EOT +echo WYDAWCA_SPOOL=\$WYDAWCA_SPOOL +echo WYDAWCA_SOURCE=\$WYDAWCA_SOURCE +echo WYDAWCA_DEST=\$WYDAWCA_DEST +echo WYDAWCA_URL=\$WYDAWCA_URL +echo WYDAWCA_TRIPLET_BASE=\$WYDAWCA_TRIPLET_BASE +echo WYDAWCA_DIST_FILE=\$WYDAWCA_DIST_FILE +exit 1 +EOT; +} +], +[wy_create_upload conversion upload/conversion-1.1.tar |