summaryrefslogtreecommitdiffabout
path: root/wikitrans
authorSergey Poznyakoff <gray@gnu.org>2018-09-01 19:10:01 (GMT)
committer Sergey Poznyakoff <gray@gnu.org>2018-09-01 19:24:02 (GMT)
commit5320bea15e388200e613e6a2bdac3c1449030986 (patch) (side-by-side diff)
tree7d8571c63f72cc690cea8323e43be09c18527c45 /wikitrans
parent0aae19835045bac0be0f22ecd0e84527cdaee21c (diff)
downloadwikitrans-5320bea15e388200e613e6a2bdac3c1449030986.tar.gz
wikitrans-5320bea15e388200e613e6a2bdac3c1449030986.tar.bz2
Bugfixes
* README.rst: Describe new options. * bin/wikitrans: Change handling of the --debug option. * wikitrans/wikimarkup.py (WikiMarkupParser): New attribute - strict. (parse_para): Don't throw UnexpectedTokenError if self.strict is False, instead ignore invalid token. (WikiMarkup): Fix Python 3 compatibility * wikitrans/wikitoken.py: Fix Python 3 compatibility
Diffstat (limited to 'wikitrans') (more/less context) (ignore whitespace changes)
-rw-r--r--wikitrans/wikimarkup.py35
-rw-r--r--wikitrans/wikitoken.py2
2 files changed, 31 insertions, 6 deletions
diff --git a/wikitrans/wikimarkup.py b/wikitrans/wikimarkup.py
index 19f69e6..1e2429f 100644
--- a/wikitrans/wikimarkup.py
+++ b/wikitrans/wikimarkup.py
@@ -145,6 +145,13 @@ class WikiMarkupParser(object):
Public attributes:
+ Input:
+ debug_level -- debug verbosity level (0 - no debug info, 100 - excessively
+ copious debug messages). Default is 0.
+ strict -- if True, parser will throw exception upon encountering
+ invalid markup tag (mostly for future use)
+
+ Output:
tree -- constructed parse tree (a subclass of WikiNode)
"""
@@ -175,6 +182,7 @@ class WikiMarkupParser(object):
tags = [ 'code', 'nowiki', 'tt', 'div', 'ref', 'references' ]
debug_level = 0
+ strict = False
def dprint(self, lev, fmt, *argv):
"""If current debug level is greater than or equal to lev, print *argv
@@ -187,7 +195,7 @@ class WikiMarkupParser(object):
inline_delims = [ "''", "'''", "[", "]", "[[", "]]", "{{", "}}", "|" ]
token_class = {}
-
+
def _new_node(self, **kwarg):
return self.token_class[kwarg['type']](self, **kwarg)
@@ -374,7 +382,7 @@ class WikiMarkupParser(object):
stack.append(i)
# Redefine all non-matched tokens as TEXT
for i in stack:
- # FIXME
+ # FIXME: How to convert node to TEXT?
self.toklist[i] = self._new_node(type='TEXT',
content=str(self.toklist[i]))
@@ -492,7 +500,10 @@ class WikiMarkupParser(object):
flush()
acc['seq'].append(self.parse_inline_delim(tok))
else:
- raise UnexpectedTokenError(tok)
+ if self.strict:
+ raise UnexpectedTokenError(tok)
+ # FIXME: Another possible variant of handling this case is to
+ # convert tok to TEXT node and append it to acc['seq']
tok = self.getkn()
flush()
if acc['seq']:
@@ -976,6 +987,13 @@ class WikiMarkup(WikiMarkupParser):
media_base=URL
Base URL for media files. Default is
'http://www.mediawiki.org/xml/export-0.3'
+
+ debug_level=INT
+ debug verbosity level (0 - no debug info, 100 - excessively
+ copious debug messages). Default is 0.
+ strict=BOOL
+ Strict parsing mode. Throw exceptions on syntax errors. Default
+ is False.
"""
self.token_class = {
'NIL': WikiNode,
@@ -999,14 +1017,17 @@ class WikiMarkup(WikiMarkupParser):
'LINK': WikiSeqNode,
'HDR': WikiHdrNode
}
-
+
for kw in keywords:
if kw == 'file':
self.file = keywords[kw]
elif kw == 'filename':
self.file = open(keywords[kw])
elif kw == 'text':
- self.text = keywords[kw].split("\n")
+ if sys.version_info[0] > 2:
+ self.text = keywords[kw].decode('utf-8').split("\n")
+ else:
+ self.text = keywords[kw].split("\n")
elif kw == 'lang':
self.lang = keywords[kw]
elif kw == 'html_base':
@@ -1015,6 +1036,10 @@ class WikiMarkup(WikiMarkupParser):
self.image_base = keywords[kw]
elif kw == 'media_base':
self.media_base = keywords[kw]
+ elif kw == 'strict':
+ self.strict = keywords[kw]
+ elif kw == 'debug_level':
+ self.debug_level = keywords[kw]
def __del__(self):
if self.file:
diff --git a/wikitrans/wikitoken.py b/wikitrans/wikitoken.py
index deedea8..1f81092 100644
--- a/wikitrans/wikitoken.py
+++ b/wikitrans/wikitoken.py
@@ -311,7 +311,7 @@ class WikiEnvNode(WikiContentNode):
return {
'envtype': self.envtype,
'level': self.level,
- 'content': map(lambda x: x.json_encode(), self.content)
+ 'content': [x for x in map(lambda x: x.json_encode(), self.content)]
}

Return to:

Send suggestions and report system problems to the System administrator.