summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergey Poznyakoff <gray@gnu.org>2018-08-16 15:45:00 +0300
committerSergey Poznyakoff <gray@gnu.org>2018-08-17 13:17:11 +0300
commit7186dbab7f1c1227e9229866e086bc417e3e4e52 (patch)
treef29114e9ff7a7b023dd3d611a9bc8808f5cf5bbd
parentd9e26129527ce84f626eb44ff95e4ecfbc5bc92a (diff)
downloadwikitrans-7186dbab7f1c1227e9229866e086bc417e3e4e52.tar.gz
wikitrans-7186dbab7f1c1227e9229866e086bc417e3e4e52.tar.bz2
Fix PEP 8 issues.
-rw-r--r--tests/test_html.py8
-rw-r--r--tests/test_texi.py6
-rw-r--r--tests/test_text.py8
-rw-r--r--tests/wikitest.py (renamed from tests/WikiTest.py)6
-rw-r--r--wikitrans/__init__.py (renamed from WikiTrans/__init__.py)0
-rw-r--r--wikitrans/wiki2html.py (renamed from WikiTrans/wiki2html.py)113
-rw-r--r--wikitrans/wiki2texi.py (renamed from WikiTrans/wiki2texi.py)61
-rw-r--r--wikitrans/wiki2text.py (renamed from WikiTrans/wiki2text.py)68
-rw-r--r--wikitrans/wikidump.py (renamed from WikiTrans/wikidump.py)41
-rw-r--r--wikitrans/wikimarkup.py (renamed from WikiTrans/wikimarkup.py)298
-rw-r--r--wikitrans/wikins.py (renamed from WikiTrans/wikins.py)0
-rw-r--r--wikitrans/wikitoken.py (renamed from WikiTrans/wikitoken.py)136
12 files changed, 545 insertions, 200 deletions
diff --git a/tests/test_html.py b/tests/test_html.py
index 3da57f6..5a15cb8 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1,14 +1,14 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import unittest
-from WikiTrans.wiki2html import HtmlWiktionaryMarkup
-from WikiTest import populateMethods
+from wikitrans.wiki2html import HtmlWikiMarkup
+from wikitest import populate_methods
-class TestWiktionaryMarkup (unittest.TestCase):
+class TestWikiMarkup (unittest.TestCase):
pass
-populateMethods(TestWiktionaryMarkup, HtmlWiktionaryMarkup, '.html')
+populate_methods(TestWikiMarkup, HtmlWikiMarkup, '.html')
if __name__ == '__main__':
unittest.main()
diff --git a/tests/test_texi.py b/tests/test_texi.py
index 75314c9..ddd26c7 100644
--- a/tests/test_texi.py
+++ b/tests/test_texi.py
@@ -1,14 +1,14 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import unittest
-from WikiTrans.wiki2texi import TexiWikiMarkup
-from WikiTest import populateMethods
+from wikitrans.wiki2texi import TexiWikiMarkup
+from wikitest import populate_methods
class TestTexiWikiMarkup (unittest.TestCase):
pass
-populateMethods(TestTexiWikiMarkup, TexiWikiMarkup, '.texi')
+populate_methods(TestTexiWikiMarkup, TexiWikiMarkup, '.texi')
if __name__ == '__main__':
unittest.main()
diff --git a/tests/test_text.py b/tests/test_text.py
index a06f519..b3d0a12 100644
--- a/tests/test_text.py
+++ b/tests/test_text.py
@@ -1,14 +1,14 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import unittest
-from WikiTrans.wiki2text import TextWiktionaryMarkup
-from WikiTest import populateMethods
+from wikitrans.wiki2text import TextWikiMarkup
+from wikitest import populate_methods
-class TestTextWiktionaryMarkup (unittest.TestCase):
+class TestTextWikiMarkup (unittest.TestCase):
pass
-populateMethods(TestTextWiktionaryMarkup, TextWiktionaryMarkup, '.text')
+populate_methods(TestTextWikiMarkup, TextWikiMarkup, '.text')
if __name__ == '__main__':
unittest.main()
diff --git a/tests/WikiTest.py b/tests/wikitest.py
index 1429f5e..ff26227 100644
--- a/tests/WikiTest.py
+++ b/tests/wikitest.py
@@ -1,13 +1,13 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
from glob import glob
import os.path
-def MarkupTest(classname, name_in, name_out):
+def wiki_markup_test(classname, name_in, name_out):
fh = open(name_out)
buf = ''.join(fh.readlines()).strip()
fh.close()
hwm = classname(filename=name_in, lang="en")
hwm.parse()
@@ -16,16 +16,16 @@ def MarkupTest(classname, name_in, name_out):
# fail
print("\n>>>%s<<<" % buf)
print(">>>%s<<<" % str(hwm).strip())
return False
-def populateMethods(cls, wcls, suffix):
+def populate_methods(cls, wcls, suffix):
def settest(self, base, wiki_name, pat_name):
def dyntest(self):
- self.assertTrue(MarkupTest(wcls, wiki_name, pat_name))
+ self.assertTrue(wiki_markup_test(wcls, wiki_name, pat_name))
meth = 'test_' + wcls.__name__ + '_' + base
dyntest.__name__ = meth
setattr(cls, meth, dyntest)
for file in glob('testdata/*.wiki'):
if os.path.isfile(file):
patfile = file[:len(file) - 5] + suffix
diff --git a/WikiTrans/__init__.py b/wikitrans/__init__.py
index 5832e38..5832e38 100644
--- a/WikiTrans/__init__.py
+++ b/wikitrans/__init__.py
diff --git a/WikiTrans/wiki2html.py b/wikitrans/wiki2html.py
index 6147642..ce65bae 100644
--- a/WikiTrans/wiki2html.py
+++ b/wikitrans/wiki2html.py
@@ -12,16 +12,27 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Wiki markup to HTML translator.
+
+Classes:
+
+HtmlWikiMarkup -- Converts Wiki material to HTML.
+HtmlWiktionaryMarkup -- Reserved for future use. Currently does the same as
+ HtmlWikiMarkup.
+
+"""
+
from __future__ import print_function
-from WikiTrans.wikimarkup import *
-from WikiTrans.wikitoken import *
-from WikiTrans.wikins import wiki_ns_re, wiki_ns
+from wikitrans.wikimarkup import *
+from wikitrans.wikitoken import *
+from wikitrans.wikins import wiki_ns_re, wiki_ns
import re
try:
from urllib import quote as url_quote
except ImportError:
from urllib.parse import quote as url_quote
@@ -76,22 +87,22 @@ class HtmlLinkNode(HtmlSeqNode):
text = self.parser.langtab[qual]
else:
tgt = self.parser.mktgt(tgt)
else:
tgt = self.parser.mktgt(arg)
return "<a href=\"%s\">%s</a>" % (tgt,
- text if (text and text != '') \
- else arg)
+ text if (text and text != '') else arg)
class HtmlRefNode(WikiRefNode):
def format(self):
target = self.ref
text = self.content.format()
- return "<a href=\"%s\">%s</a>" % (target,
- text if (text and text != '') \
- else target)
+ return "<a href=\"%s\">%s</a>" % (
+ target,
+ text if (text and text != '') else target
+ )
class HtmlFontNode(HtmlSeqNode):
def format(self):
comm = { 'IT': 'i',
'BOLD': 'b' }
s = '<%s>' % comm[self.type]
@@ -149,20 +160,20 @@ class HtmlTagNode(WikiTagNode):
elif self.tag == 'references':
s = '<div class="references">\n'
s += '<ol class="references">\n'
n = 0
for ref in self.parser.references:
n += 1
- s += ('<li id="cite_note-%d">' + \
- '<span class="mw-cite-backlink">' + \
- '<b><a href="#cite_ref-%d">^</a></b>' + \
- '</span>' + \
- '<span class="reference-text">' + \
- ref.content.format() + \
- '</span>' + \
- '</li>\n') % (n,n)
+ s += ('<li id="cite_note-%d">'
+ + '<span class="mw-cite-backlink">'
+ + '<b><a href="#cite_ref-%d">^</a></b>'
+ + '</span>'
+ + '<span class="reference-text">'
+ + ref.content.format()
+ + '</span>'
+ + '</li>\n') % (n,n)
s += '</ol>\n</div>\n'
return s
else:
s = '<' + self.tag
if self.args:
s += ' ' + str(self.args)
@@ -185,22 +196,54 @@ class HtmlPreNode(HtmlSeqNode):
class HtmlIndNode(WikiIndNode):
def format(self):
return ("<dl><dd>" * self.level) + self.content.format() + "</dd></dl>" * self.level
class HtmlWikiMarkup(WikiMarkup):
- """
- A (hopefully) general-purpose Wiki->HTML translator class.
- FIXME: 1. See WikiMarkup for a list
- 2. [[official position]]s : final 's' gets after closing </a> tag.
- Should be before.
+ """A Wiki markup to HTML translator class.
+
+ Usage:
+
+ x = HtmlWikiMarkup(file="input.wiki")
+ # Parse the input:
+ x.parse()
+ # Print it as HTML:
+ print(str(x))
+
+ Known bugs:
+ * [[official position]]s
+ Final 's' gets after closing </a> tag. Should be before.
"""
nested = 0
references = []
def __init__(self, *args, **kwargs):
+ """Create a HtmlWikiMarkup object.
+
+ Arguments:
+
+ filename=FILE
+ Read Wiki material from the file named FILE.
+ file=FD
+ Read Wiki material from file object FD.
+ text=STRING
+ Read Wiki material from STRING.
+ lang=CODE
+ Specifies source language. Default is 'en'. This variable can be
+ referred to as '%(lang)s' in the keyword arguments below.
+ html_base=URL
+ Base URL for cross-references. Default is
+ 'http://%(lang)s.wiktionary.org/wiki/'
+ image_base=URL
+ Base URL for images. Default is
+ 'http://upload.wikimedia.org/wikipedia/commons/thumb/a/bf'
+ media_base=URL
+ Base URL for media files. Default is
+ 'http://www.mediawiki.org/xml/export-0.3'
+ """
+
super(HtmlWikiMarkup, self).__init__(*args, **kwargs)
self.token_class['LINK'] = HtmlLinkNode
self.token_class['TMPL'] = HtmlLinkNode
self.token_class['REF'] = HtmlRefNode
self.token_class['IT'] = HtmlFontNode
self.token_class['BOLD'] = HtmlFontNode
@@ -268,32 +311,10 @@ class HtmlWikiMarkup (WikiMarkup):
str = ""
for elt in self.tree:
str += elt.format()
return str
class HtmlWiktionaryMarkup(HtmlWikiMarkup):
- """
- A class for translating Wiktionary articles into HTML.
- This version does not do much, except that it tries to correctly
- format templates. But "tries" does not mean "does". The heuristics
- used here is clearly not enough to cope with it.
-
- 1. FIXME:
- The right solution would be to have a database of templates with their
- semantics and to decide on their rendering depending on that. E.g.
- {{term}} in en.wiktionary means "replace this with the search term".
- This, however, does not work in other wiktionaries. There are
- also more complex templates, e.g.: {{t+|bg|врата|n|p|tr=vrata|sc=Cyrl}}
- I don't know what it means. Couldn't find any documentation either.
- Again, this template does not work in other dictionaries.
-
- 2. Capitulation notice:
- Given the:
- 1. vast amount of wiktionaries available,
- 2. abundance of various templates for each wictionary,
- 3. apparent lack of documentation thereof,
- 4. the lack of standardized language-independent templates,
- I dont see any way to cope with the template-rendering task within a
- reasonable amount of time.
-
- Faeci quod potui, faciant meliora potentes.
+ """A class for translating Wiktionary articles into HTML.
+
+ Reserved for future use. Currently does the same as HtmlWikiMarkup.
"""
diff --git a/WikiTrans/wiki2texi.py b/wikitrans/wiki2texi.py
index 7297195..d9e5f52 100644
--- a/WikiTrans/wiki2texi.py
+++ b/wikitrans/wiki2texi.py
@@ -12,15 +12,24 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from WikiTrans.wikimarkup import *
-from WikiTrans.wikitoken import *
-from WikiTrans.wikins import wiki_ns_re, wiki_ns
+"""
+Wiki markup to Texinfo translator.
+
+Classes:
+
+TexiWikiMarkup -- Converts Wiki material to Texinfo.
+
+"""
+
+from wikitrans.wikimarkup import *
+from wikitrans.wikitoken import *
+from wikitrans.wikins import wiki_ns_re, wiki_ns
import re
import urllib
class Acc(list):
def prepend(self,x):
self.insert(0,x)
@@ -249,12 +258,24 @@ class TexiRefNode(WikiRefNode):
if text and text != '':
parser._print("@uref{%s,%s}" % (target, text), escape=False)
else:
parser._print("@uref{%s}" % target, escape=False)
class TexiWikiMarkup(WikiMarkup):
+ """Wiki markup to Texinfo translator class.
+
+ Usage:
+
+ x = TexiWikiMarkup(file="input.wiki")
+ # Parse the input:
+ x.parse()
+ # Print it as Texi:
+ print(str(x))
+
+ """
+
nested = 0
sectcomm = {
'numbered': [
'@top',
'@chapter',
'@section',
@@ -285,12 +306,46 @@ class TexiWikiMarkup (WikiMarkup):
}
sectioning_model = 'numbered'
sectioning_start = 0
def __init__(self, *args, **keywords):
+ """Create a TexiWikiMarkup object.
+
+ Arguments:
+
+ filename=FILE
+ Read Wiki material from the file named FILE.
+ file=FD
+ Read Wiki material from file object FD.
+ text=STRING
+ Read Wiki material from STRING.
+
+ sectioning_model=MODEL
+ Select the Texinfo sectioning model for the output document. Possible
+ values are:
+
+ 'numbered'
+ Top of document is marked with "@top". Headings ("=", "==",
+ "===", etc) produce "@chapter", "@section", "@subsection", etc.
+ 'unnumbered'
+ Unnumbered sectioning: "@top", "@unnumbered", "@unnumberedsec",
+ "@unnumberedsubsec".
+ 'appendix'
+ Sectioning suitable for appendix entries: "@top", "@appendix",
+ "@appendixsec", "@appendixsubsec", etc.
+ 'heading'
+ Use heading directives to reflect sectioning: "@majorheading",
+ "@chapheading", "@heading", "@subheading", etc.
+ sectioning_start=N
+ Shift resulting heading level by N positions. For example, supposing
+ "sectioning_model='numbered'", "== A ==" normally produces
+ "@section A" on output. Now, if given "sectioning_start=1", this
+ directive will produce "@subsection A" instead.
+ """
+
super(TexiWikiMarkup, self).__init__(*args, **keywords)
self.token_class['TEXT'] = TexiTextNode
self.token_class['TAG'] = TexiTagNode
self.token_class['PARA'] = TexiParaNode
self.token_class['PRE'] = TexiPreNode
diff --git a/WikiTrans/wiki2text.py b/wikitrans/wiki2text.py
index cb3a183..1fbc61b 100644
--- a/WikiTrans/wiki2text.py
+++ b/wikitrans/wiki2text.py
@@ -12,15 +12,26 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from WikiTrans.wikitoken import *
-from WikiTrans.wikimarkup import *
-from WikiTrans.wikins import wiki_ns_re, wiki_ns
+"""
+Wiki markup to plain text translator.
+
+Classes:
+
+TextWikiMarkup -- Converts Wiki material to plain text.
+TextWiktionaryMarkup -- Reserved for future use. Currently does the same as
+ TextWikiMarkup.
+
+"""
+
+from wikitrans.wikitoken import *
+from wikitrans.wikimarkup import *
+from wikitrans.wikins import wiki_ns_re, wiki_ns
import re
try:
from urllib import quote as url_quote
except ImportError:
from urllib.parse import quote as url_quote
@@ -104,15 +115,15 @@ class TextLinkNode(WikiSeqNode):
ns = self.parser.wiki_ns_name(qual)
if ns:
if ns == 'NS_IMAGE':
if not self.parser.show_urls:
return ""
text = "[%s: %s]" % (qual, text if text else arg)
- tgt = self.image_base + '/' + \
- url_quote(tgt) + \
- '/250px-' + url_quote(tgt)
+ tgt = "%s/%s/250px-%s" % (self.image_base,
+ url_quote(tgt),
+ url_quote(tgt))
elif ns == 'NS_MEDIA':
text = "[%s]" % (qual)
else:
tgt = self.parser.mktgt(tgt)
elif self.type == 'LINK' and qual in self.parser.langtab:
text = self.parser.langtab[qual] + ": " + tgt
@@ -138,14 +149,17 @@ class TextBarNode(WikiNode):
if w < 5:
w = 5
return "\n" + ("-" * (w - 5)).center(w - 1) + "\n"
class TextHdrNode(WikiHdrNode):
def format(self):
- return "\n" + ("*" * self.level) + " " + \
- self.content.format().lstrip(" ") + "\n\n"
+ return ("\n"
+ + ("*" * self.level)
+ + " "
+ + self.content.format().lstrip(" ")
+ + "\n\n")
class TextRefNode(WikiRefNode):
def format(self):
text = self.content.format()
if text:
return "%s (see %s) " % (text, self.ref)
@@ -202,14 +216,22 @@ class TextTagNode(WikiTagNode):
s += ' ' + str(self.args)
s += '>' + self.content.format() + '</' + self.tag + '>'
return s
class TextWikiMarkup(WikiMarkup):
- """
- A (general-purpose Wiki->Text translator class.
+ """A Wiki markup to plain text translator.
+
+ Usage:
+
+ x = TextWikiMarkup(file="input.wiki")
+ # Parse the input:
+ x.parse()
+ # Print it as plain text:
+ print(str(x))
+
"""
# Output width
width = 78
# Do not show references.
show_urls = False
@@ -220,12 +242,31 @@ class TextWikiMarkup (WikiMarkup):
num = 0
# Array of footnote references
references = []
def __init__(self, *args, **keywords):
+ """Create a TextWikiMarkup object.
+
+ Arguments:
+
+ filename=FILE
+ Read Wiki material from the file named FILE.
+ file=FD
+ Read Wiki material from file object FD.
+ text=STRING
+ Read Wiki material from STRING.
+
+ width=N
+ Limit output width to N columns. Default is 78.
+ show_urls=False
+ By default, the link URLs are displayed in parentheses next to the
+ link text. If this argument is given, only the link text will be
+ displayed.
+ """
+
super(TextWikiMarkup,self).__init__(*args, **keywords)
if 'width' in keywords:
self.width = keywords['width']
if 'show_urls' in keywords:
self.show_urls = keywords['show_urls']
self.token_class['SEQ'] = TextSeqNode
@@ -296,11 +337,12 @@ class TextWikiMarkup (WikiMarkup):
str = ""
for elt in self.tree:
str += elt.format()
return str
class TextWiktionaryMarkup(TextWikiMarkup):
+ """A class for translating Wiktionary articles into plain text.
+
+ Reserved for future use. Currently does the same as TextWikiMarkup.
"""
- See documentation for HtmlWiktionaryMarkup
- """
- # FIXME: It is supposed to do something about templates
+
diff --git a/WikiTrans/wikidump.py b/wikitrans/wikidump.py
index 7457dfa..d5f651c 100644
--- a/WikiTrans/wikidump.py
+++ b/wikitrans/wikidump.py
@@ -11,32 +11,67 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Print Wiki parse tree as JSON.
+
+Classes:
+
+DumpWikiMarkup
+
+"""
+
from __future__ import print_function
-from WikiTrans.wikitoken import *
+from wikitrans.wikitoken import *
import json
-from WikiTrans.wikimarkup import WikiMarkup
+from wikitrans.wikimarkup import WikiMarkup
class DumpReferences(object):
idx = 0
def __len__(self):
return self.idx + 1
def append(self, obj):
self.idx += 1
class DumpWikiMarkup(WikiMarkup):
+ """Produce a JSON dump of the Wiki markup parse tree.
+
+ Usage:
+
+ x = DumpWikiMarkup(file="input.wiki")
+ # Parse the input:
+ x.parse()
+ # Print a JSON dump of the parse tree
+ print(str(x))
+
+ """
+
indent = None
references = DumpReferences()
def __init__(self, **kwarg):
+ """Create a DumpWikiMarkup object.
+
+ Arguments:
+
+ filename=FILE
+ Read Wiki material from the file named FILE.
+ file=FD
+ Read Wiki material from file object FD.
+ text=STRING
+ Read Wiki material from STRING.
+ indent=N
+ Basic indent offset for JSON objects.
+ """
+
n = kwarg.pop('indent', None)
if n != None:
self.indent = int(n)
- WikiMarkup.__init__(self, **kwarg)
+ super(DumpWikiMarkup,self).__init__(self, **kwarg)
def __str__(self):
return json.dumps(self.tree,
cls=WikiNodeEncoder,
indent=self.indent,
separators=(',',': '),
sort_keys=True)
diff --git a/WikiTrans/wikimarkup.py b/wikitrans/wikimarkup.py
index 6cbf5de..77c3b30 100644
--- a/WikiTrans/wikimarkup.py
+++ b/wikitrans/wikimarkup.py
@@ -12,32 +12,57 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Wiki markup parser.
+
+This module provides two class:
+
+WikiMarkupParser:
+ An abstract parser class, which serves as a base class for all markup
+ classes in this package.
+
+WikiMarkup
+ A subclass of the above, providing basic input method.
+
+"""
+
from __future__ import print_function
import sys
import re
from types import *
-from WikiTrans.wikitoken import *
+from wikitrans.wikitoken import *
-__all__ = [ "BaseWikiMarkup", "WikiMarkup",
- "TagAttributes", "TagAttributeSyntax" ]
+__all__ = [ "WikiMarkupParser", "WikiMarkup",
+ "TagAttributes", "TagAttributeSyntaxError" ]
-class UnexpectedToken(Exception):
+class UnexpectedTokenError(Exception):
def __init__(self, value):
self.value = value
-class TagAttributeSyntax(Exception):
+class TagAttributeSyntaxError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class TagAttributes(object):
+ """A dictionary-like collection of tag attributes.
+
+ Example:
+
+ attr = TagAttributes('href="foo" length=2')
+ if 'href' in attr:
+ print(x['href']) # returns "foo"
+ for a in attr:
+ ...
+ """
+
attrstart = re.compile("^(?P<attr>[a-zA-Z0-9_-]+)(?P<eq>=\")?")
valseg = re.compile("^[^\\\"]+")
tab = {}
printable = None
def __init__(self, string):
if not string:
@@ -65,13 +90,13 @@ class TagAttributes(object):
s = s[1:]
break
else:
val = 1
self.tab[name] = val
else:
- raise TagAttributeSyntax(s)
+ raise TagAttributeSyntaxError(s)
def __len__(self):
return len(self.tab)
def __getitem__(self, key):
return self.tab[key]
def __contains__(self, key):
return key in self.tab
@@ -86,13 +111,32 @@ class TagAttributes(object):
del self.tab[key]
def __str__(self):
return self.printable
def __repr__(self):
return self.printable
-class BaseWikiMarkup(object):
+class WikiMarkupParser(object):
+ """Parser for Wiki markup language.
+
+ Given input in Wiki markup language creates an abstract parse tree for it.
+ This is a base class for actual parsers. The subclasses must provide the
+ input method.
+
+ Public methods:
+
+ parse() -- parse the input.
+
+ Abstract methods (must be overridden by the subclass):
+
+ input() -- returns next physical line from the input material.
+
+ Public attributes:
+
+ tree -- constructed parse tree (a subclass of WikiNode)
+
+ """
delim = re.compile("^==+[ \t]*|[ \t]*==+[ \t]*$|(^----$)|^\\*+|^#+|^[;:]+|(\\[\\[)|\\[|(\\{\\{)|(\\]\\])|\\]|(\\}\\})|\\||(\\'\\'\\'?)|<")
otag = re.compile("<(?P<tag>[a-zA-Z0-9_]+)(?:\s+(?P<args>[^/][^>]+))?\s*(?P<closed>/)?>")
ctag = re.compile("</(?P<tag>[a-zA-Z0-9_]+)\s*>")
refstart = re.compile("^https?://")
@@ -116,12 +160,15 @@ class BaseWikiMarkup(object):
tags = [ 'code', 'nowiki', 'tt', 'div', 'ref', 'references' ]
debug_level = 0
def dprint(self, lev, fmt, *argv):
+ """If current debug level is greater than or equal to lev, print *argv
+ according to format.
+ """
if self.debug_level >= lev:
for l in (fmt % argv).split('\n'):
print("[DEBUG] %s" % l)
inline_delims = [ "''", "'''", "[", "]", "[[", "]]", "{{", "}}", "|" ]
@@ -145,41 +192,42 @@ class BaseWikiMarkup(object):
'ELT': WikiEltNode,
'ENV': WikiEnvNode,
'LINK': WikiSeqNode,
'HDR': WikiHdrNode
}
- def __createWikiNode(self,**kwarg):
+ def _new_node(self,**kwarg):
return self.token_class[kwarg['type']](self, **kwarg)
def tokread(self):
+ """Read next token from the input. Return it as a subclass of WikiNode."""
line = None
pos = 0
while 1:
if (not line or pos == len(line)):
try:
line = self.input()
pos = 0
except StopIteration:
line = u''
if not line or line == "":
- yield(self.__createWikiNode(type='NIL'))
+ yield(self._new_node(type='NIL'))
break
if line == '\n':
- yield(self.__createWikiNode(type='NL'))
+ yield(self._new_node(type='NL'))
line = None
continue
self.dprint(100, "LINE: %s", line[pos:])
m = self.delim.search(line, pos)
if m:
if (pos < m.start(0)):
- yield(self.__createWikiNode(type='TEXT',
+ yield(self._new_node(type='TEXT',
content=line[pos:m.start(0)]))
pos = m.start(0)
t = None
if line[m.start(0)] == '<':
m = self.otag.match(line, pos)
@@ -188,95 +236,106 @@ class BaseWikiMarkup(object):
if m.group('tag') == 'nowiki':
if not m.group('closed'):
while 1:
try:
m = self.ctag.search(line, pos)
if m and m.group('tag') == 'nowiki':
- yield(self.__createWikiNode(type='TEXT',
+ yield(self._new_node(type='TEXT',
content=line[pos:m.start(0)] ))
pos = m.end(0)
break
- yield(self.__createWikiNode(type='TEXT',
+ yield(self._new_node(type='TEXT',
content=line[pos:]))
line = self.input()
pos = 0
except StopIteration:
break
continue
elif m.group('tag') in self.tags:
try:
- yield(self.__createWikiNode(type='OTAG',
+ yield(self._new_node(type='OTAG',
tag=m.group('tag'),
isblock=(line[pos] == '\n'),
args=TagAttributes(m.group('args'))))
if m.group('closed'):
- yield(self.__createWikiNode(type='CTAG',
+ yield(self._new_node(type='CTAG',
tag=m.group('tag')))
- except TagAttributeSyntax:
- yield(self.__createWikiNode(type='TEXT',content=m.group(0)))
+ except TagAttributeSyntaxError:
+ yield(self._new_node(type='TEXT',
+ content=m.group(0)))
continue
else:
- yield(self.__createWikiNode(type='TEXT',content=m.group(0)))
+ yield(self._new_node(type='TEXT',content=m.group(0)))
continue
else:
m = self.ctag.match(line, pos)
if m:
if m.group('tag') in self.tags:
- yield(self.__createWikiNode(type='CTAG',
+ yield(self._new_node(type='CTAG',
tag=m.group('tag')))
pos = m.end(0)
continue
else:
- yield(self.__createWikiNode(type='TEXT',
+ yield(self._new_node(type='TEXT',
content=line[pos:pos+1]))
pos += 1
continue
else:
pos = m.end(0)
content = m.group(0)
if content[0] in self.envtypes:
- node = self.__createWikiNode(type='DELIM',
+ node = self._new_node(type='DELIM',
content=content,
isblock=True,
continuation=pos < len(line) and line[pos] == ":")
if node.continuation:
node.content += node.content[0]
pos += 1
yield(node)
while pos < len(line) and line[pos] in [' ', '\t']:
pos += 1
else:
- yield(self.__createWikiNode(type='DELIM',
+ yield(self._new_node(type='DELIM',
isblock=(content.strip() not in self.inline_delims),
content=content.strip()))
continue
if line:
if line[-1] == '\n':
if line[pos:-1] != '':
- yield(self.__createWikiNode(type='TEXT',content=line[pos:-1]))
- yield(self.__createWikiNode(type='NL'))
+ yield(self._new_node(type='TEXT',content=line[pos:-1]))
+ yield(self._new_node(type='NL'))
else:
- yield(self.__createWikiNode(type='TEXT',content=line[pos:]))
+ yield(self._new_node(type='TEXT',content=line[pos:]))
line = None
def input(self):
+ """Return next physical line from the input.
+
+ This method must be overridden by the subclass.
+ """
return None
def swaptkn(self, i, j):
+ """Swap tokens at indices i and j in toklist."""
self.dprint(80, "SWAPPING %s <-> %s", i, j)
x = self.toklist[i]
self.toklist[i] = self.toklist[j]
self.toklist[j] = x
def tokenize(self):
+ """Tokenize the input.
+
+ Read tokens from the input (supplied by the input() method). Place the