Bugfixes

* README.rst: Describe new options.
* bin/wikitrans: Change handling of the --debug option.
* wikitrans/wikimarkup.py (WikiMarkupParser): New attribute - strict.
(parse_para): Don't throw UnexpectedTokenError if self.strict is False,
instead ignore invalid token.
(WikiMarkup): Fix Python 3 compatibility
* wikitrans/wikitoken.py: Fix Python 3 compatibility

4 files changed, 42 insertions, 6 deletions

diff --git a/README.rst b/README.rst
index 7838fa1..7edeb04 100644
--- a/README.rst
+++ b/README.rst
@@ -39,2 +39,8 @@ media_base = *url*
 
+debug_level = *int*
+  Debug verbosity level (0 - no debug info, 100 - excessively
+  copious debug messages). Default is 0.
+
+strict = *bool*
+  Strict parsing mode. Throw exceptions on syntax errors. Default is False.
 
diff --git a/bin/wikitrans b/bin/wikitrans
index 4a0fc06..09ba0b3 100755
--- a/bin/wikitrans
+++ b/bin/wikitrans
@@ -73,2 +73,7 @@ def setkw(option, opt, value, parser):
 
+def setdebug(option, opt, value, parser):
+    if not parser.values.kwdict:
+        parser.values.kwdict = {}
+    parser.values.kwdict['debug_level'] = value
+
 def getwiki(url, options):
@@ -137,4 +142,4 @@ Otherwise, ARG is name of the file to read wiki material from.
     parser.add_option('-d', '--debug',
-                      action='store', type='int', dest='debug',
-                      default=0,
+                      action='callback', callback=setdebug,
+                      type='int', dest='kwdict',
                       help='set debug level (0..100)')
@@ -152,3 +157,4 @@ Otherwise, ARG is name of the file to read wiki material from.
         if options.base_url:
-            getwiki(options.base_url + '/wiki/Special:Export/' + args[0], options)
+            getwiki(options.base_url + '/wiki/Special:Export/' + args[0],
+                    options)
         elif args[0] == '-':
@@ -169,3 +175,2 @@ Otherwise, ARG is name of the file to read wiki material from.
             markup = handlers[options.otype][options.itype](**options.kwdict)
-            markup.debug_level = options.debug
             markup.parse()
diff --git a/wikitrans/wikimarkup.py b/wikitrans/wikimarkup.py
index 19f69e6..1e2429f 100644
--- a/wikitrans/wikimarkup.py
+++ b/wikitrans/wikimarkup.py
@@ -147,2 +147,9 @@ class WikiMarkupParser(object):
 
+    Input:
+    debug_level -- debug verbosity level (0 - no debug info, 100 - excessively
+                   copious debug messages). Default is 0.
+    strict      -- if True, parser will throw exception upon encountering
+                   invalid markup tag (mostly for future use)
+
+    Output:
     tree     --  constructed parse tree (a subclass of WikiNode)
@@ -177,2 +184,3 @@ class WikiMarkupParser(object):
     debug_level = 0
+    strict = False
 
@@ -376,3 +384,3 @@ class WikiMarkupParser(object):
         for i in stack:
-            # FIXME
+            # FIXME: How to convert node to TEXT?
             self.toklist[i] = self._new_node(type='TEXT',
@@ -494,3 +502,6 @@ class WikiMarkupParser(object):
             else:
+                if self.strict:
                     raise UnexpectedTokenError(tok)
+                # FIXME: Another possible variant of handling this case is to
+                # convert tok to TEXT node and append it to acc['seq']
             tok = self.getkn()
@@ -978,2 +989,9 @@ class WikiMarkup(WikiMarkupParser):
               'http://www.mediawiki.org/xml/export-0.3'
+
+        debug_level=INT
+          debug verbosity level (0 - no debug info, 100 - excessively
+                                 copious debug messages). Default is 0.
+        strict=BOOL
+          Strict parsing mode. Throw exceptions on syntax errors. Default
+          is False.
         """
@@ -1008,2 +1026,5 @@ class WikiMarkup(WikiMarkupParser):
             elif kw == 'text':
+                if sys.version_info[0] > 2:
+                    self.text = keywords[kw].decode('utf-8').split("\n")
+                else:
                     self.text = keywords[kw].split("\n")
@@ -1017,2 +1038,6 @@ class WikiMarkup(WikiMarkupParser):
                 self.media_base = keywords[kw]
+            elif kw == 'strict':
+                self.strict = keywords[kw]
+            elif kw == 'debug_level':
+                self.debug_level = keywords[kw]
 
diff --git a/wikitrans/wikitoken.py b/wikitrans/wikitoken.py
index deedea8..1f81092 100644
--- a/wikitrans/wikitoken.py
+++ b/wikitrans/wikitoken.py
@@ -313,3 +313,3 @@ class WikiEnvNode(WikiContentNode):
             'level': self.level,
-            'content': map(lambda x: x.json_encode(), self.content)
+            'content': [x for x in map(lambda x: x.json_encode(), self.content)]
         }