Improvements in mailsync

* bin/mailsync: Change configuration. Improve error handling.
Read list names from SQL, unless core.listfile is set.
* install.pl: Install manpages.
* man/mailsync.1: New file.

1 files changed, 130 insertions, 44 deletions

diff --git a/bin/mailsync b/bin/mailsync
index 555b4da..00e6d2c 100755
--- a/bin/mailsync
+++ b/bin/mailsync
@@ -1,5 +1,5 @@
 #! /usr/bin/python
-# Copyright (C) 2014 Sergey Poznyakoff
+# Copyright (C) 2014-2015 Sergey Poznyakoff
 #  
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,33 +17,72 @@
 from __future__ import print_function
 import os
 import sys
+import errno
 import getopt
 import mailbox
 import re
+import MySQLdb
 from time import strftime, strptime
 from subprocess import Popen,PIPE,call
 
-rootdir = "/srv/mailman/var"
 dbg = 0
 dry_run = 0
 reset_state = 0
 
-conf = { 'etcdir' : "/srv/mailman/etc",
-         'listfile': "/etc/mail/mailman/SAVANE",
-         'indexrootdir': rootdir + "/namazu/index",
-         'mboxrootdir': rootdir + "/archives/private",
-         'htmlrootdir': rootdir + "/archives/html",
-         'base_url': "http://mail.gnu.org.ua/mailman/listarchive"
-       }
+EX_OK = 0
+EX_FAILURE = 1
+EX_USAGE = 64
+EX_CONFIG = 78
+
+kwtab = {
+    'core': {
+        'rootdir': False,
+        'etcdir' : True,
+        'listfile': False,
+        'indexrootdir': True,
+        'mboxrootdir': True,
+        'htmlrootdir': True,
+        'base_url': True,
+    },
+    'mysql': {
+        'host': False,
+        'user': False,
+        'passwd': False,
+        'db': False,
+        'port': False,
+        'unix_socket': False,
+        'connect_timeout': False,
+        'init_command': False,
+        'read_default_file': False,
+        'read_default_group': False,
+        'use_unicode': False,
+        'charset': False,
+        'sql_mode': False,
+        #ssl: FIXME
+    }
+}    
+
+conf = { }
 
 os.environ["PATH"] = "/srv/mailman/bin:/usr/local/bin:" + os.environ["PATH"] 
 
+status = EX_OK
+
 def debug(lev, text):
     if (dbg >= lev):
         print("debug: %s" % text, file=sys.stderr)
 
+def absdir(dir):
+    if dir.startswith('/'):
+        return dir
+    elif 'rootdir' in conf['core']:
+        return conf['core']['rootdir'] + '/' + dir
+    else:
+        print("core.rootdir not set in config", file=sys.stderr)
+        exit(EX_CONFIG)
+        
 def get_state(listid):
-    file = conf['htmlrootdir'] + "/" + listid + "/.state"
+    file = absdir(conf['core']['htmlrootdir'] + "/" + listid + "/.state")
     try:
         with open(file, 'r') as f:
             r = int(f.readline().rstrip())
@@ -52,24 +91,34 @@ def get_state(listid):
     return r
 
 def save_state(listid,n):
-    file = conf['htmlrootdir'] + "/" + listid + "/.state"
+    file = absdir(conf['core']['htmlrootdir'] + "/" + listid + "/.state")
     try:
         with open(file, 'w') as f:
             f.write(str(n + 1) + "\n")
-    except:
+    except IOError:
         print("cannot write %s: %s" % (file,sys.exc_info()), file=sys.stderr)
+        status = EX_FAILURE
 
 def save_timestamp(listid,ts):
-    file = conf['htmlrootdir'] + "/" + listid + "/.timestamp"
+    file = absdir(conf['core']['htmlrootdir'] + "/" + listid + "/.timestamp")
     try:
         with open(file, 'w') as f:
             f.write(strftime('%s', ts) + "\n")
-    except:
+    except IOError:
         print("cannot write %s: %s" % (file,sys.exc_info()), file=sys.stderr)
+        status = EX_FAILURE
+
 
 def mboxtohtml(listid):
-    name = conf['mboxrootdir'] + "/" + listid + ".mbox/" + listid + ".mbox"
-    inbox = mailbox.mbox(name)
+    name = absdir(conf['core']['mboxrootdir'] + "/" + listid + ".mbox/" + listid + ".mbox")
+    try:
+        inbox = mailbox.mbox(name)
+    except IOError as e:
+        if e.errno == errno.ENOENT:
+            print("%s: archive does not exist" % (name))
+            return False
+        raise
+            
     if reset_state:
         n = 0
     else:
@@ -82,17 +131,17 @@ def mboxtohtml(listid):
         t = strptime(s[1], '%a %b %d %H:%M:%S %Y')
         period = strftime('%Y-%m', t)
         periodh = strftime('%B, %Y', t)
-        dirname = conf['htmlrootdir'] + "/" + listid + "/" + period
+        dirname = absdir(conf['core']['htmlrootdir'] + "/" + listid + "/" + period)
         if not os.path.exists(dirname):
             os.makedirs(dirname, 02775)
 
         cmd = [ 'mhonarc',
                 '-add',
-                '-rcfile', conf['etcdir'] + '/main.mrc',
+                '-rcfile', absdir(conf['core']['etcdir'] + '/main.mrc'),
                 '-outdir', dirname,
                 '-definevar', 'gray_period=' + period,
                 "-definevar", 'gray_period_pretty="' + periodh + '"',
-                '-definevar', 'gray_base_url="' + conf['base_url'] + '"',
+                '-definevar', 'gray_base_url="' + conf['core']['base_url'] + '"',
                 '-definevar', 'gray_listname=' + listid ];
         debug(1, "running %s" % str(cmd))
         if not dry_run:
@@ -101,30 +150,43 @@ def mboxtohtml(listid):
             if p.returncode < 0:
                 print("command \"%s\" exited on signal %s" %
                       (str(cmd), -p.returncode), file=sys.stderr)
+                status = EX_FAILURE
             elif p.returncode:
                 print("command \"%s\" exited with code %s" %
                       (str(cmd), p.returncode), file=sys.stderr)
+                status = EX_FAILURE
             if err:
                 print("\"%s\": %s" % (str(cmd), err), file=sys.stderr)
+                status = EX_FAILURE
                         
     save_state(listid, n)
     if t > 0:
 	save_timestamp(listid, t)
+    
+    return True
         
-def savane_lists(file):
-    with open(file, 'r') as f:
-        for line in f:
-            r = line.strip()
-            if len(r) != 0 and r[0] != '#':
-                yield(r)
+def savane_lists(file = None):
+    if 'listfile' in conf['core']:
+        with open(conf['core']['listfile'], 'r') as f:
+            for line in f:
+                r = line.strip()
+                if len(r) != 0 and r[0] != '#':
+                    yield(r)
+    else:
+        db = MySQLdb.connect(**conf['mysql'])
+        cur = db.cursor()
+        cur.execute("select list_name from mail_group_list where status=5")
+        for row in cur.fetchall():
+            yield(row[0])
+        db.close
 
 def reindex(listid):
-    outdir = conf['indexrootdir'] + "/" + listid
+    outdir = absdir(conf['core']['indexrootdir'] + "/" + listid)
     if not os.path.exists(outdir):
         os.makedirs(outdir, 02775)
-    indir = conf['htmlrootdir'] + "/" + listid
+    indir = absdir(conf['core']['htmlrootdir'] + "/" + listid)
     if os.path.exists(indir):
-        cmd = '/srv/mailman/bin/mknmz -q -O ' + outdir + \
+        cmd = 'mknmz -q -O ' + outdir + \
               ' --allow="msg[0-9][0-9]*\.html" ' + indir
         debug(1, "running %s" % str(cmd))
         if not dry_run:
@@ -132,29 +194,51 @@ def reindex(listid):
             r = p.wait()
             if r != 0:
                 print("command '%s' exited with status %s" % (cmd,r), file=sys.stderr)
+                status = EX_FAILURE
 
 def mailsync(l):
-    mboxtohtml(l)
-    reindex(l)
+    if mboxtohtml(l):
+        reindex(l)
 
 def read_config(file):
     debug(1, "reading configuration from %s" % file)
     lno = 0
+    section = None
+    err = 0
     with open(file, 'r') as f:
         for line in f:
             lno += 1
             r = line.strip()
-            if len(r) != 0 and r[0] != '#':
-                (kw,val) = re.split('\s+', r, 1)
-                if kw in conf.keys():
-                    conf[kw] = val
-                else:
+            if r.startswith('[') and r.endswith(']'):
+                section = r[1:-1]
+                if not section in kwtab:
+                    print("%s:%s: unknown section" % (file,lno),
+                          file=sys.stderr)
+                    err += 1
+                conf[section] = {}
+            elif len(r) != 0 and r[0] != '#':
+                if not section:
+                    print("%s:%s: statement out of section" % (file,lno),
+                          file=sys.stderr)
+                    continue
+
+                (kw,val) = re.split('\s*=\s*', r, 1)
+                if not kw in kwtab[section].keys():
                     print("%s:%s: unknown keyword" % (file,lno),
                           file=sys.stderr)
-        if dbg > 1:
-            debug(2, "Configuration dump:")
-            for k in conf.keys():
-                debug(2, "%s=%s" % (k, conf[k]))
+                    err += 1
+                conf[section][kw] = val
+
+        if not 'core' in conf:
+            print("%s: [core] section missing" % (file), file=sys.stderr)
+            err += 1
+        for s in kwtab.keys():
+            for k in kwtab[s].keys():
+                if kwtab[s][k] and not k in conf[s]:
+                    print("%s: %s.%s not set" % (file,s,k), file=sys.stderr)
+                    err += 1
+        if err:
+            exit(EX_CONFIG)
 
 def usage(code=0):
     if code:
@@ -176,13 +260,13 @@ OPTIONS are:
     exit(code)
                 
 if __name__ == '__main__':
-    config = ''
+    config = '/etc/mailsync.conf'
     try:
         opts, args = getopt.getopt(sys.argv[1:], "dhc:nr",
                                    ["debug", "help", "config=", "dry-run",
                                     "reset"])
     except getopt.GetoptError:
-        usage(1)
+        usage(EX_USAGE)
 
     for o, a in opts:
         if o in ("-h", "--help"):
@@ -199,16 +283,18 @@ if __name__ == '__main__':
     if dry_run:
         dbg += 1
 
-    if config != '':
-        read_config(config)
+    read_config(config)
 
     if len(args) > 0:
         for l in args:
             mailsync(l)
     else:
-        for l in savane_lists(conf['listfile']):
+        for l in savane_lists():
             mailsync(l)
 
+    exit(status)
+
+