# This file is part of python-ly, https://pypi.python.org/pypi/python-ly
#
# Copyright (c) 2008 - 2015 by Wilbert Berendsen
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
# See http://www.gnu.org/licenses/ for more information.
"""
Token base classes.
Don't use this module directly!
All important classes are imported in the ly.lex namespace.
You can, however, import _token in mode modules that also are in this directory.
This module starts with an underscore so that its name does not interfere with
the mode modules.
"""
from __future__ import unicode_literals
from .. import slexer
[docs]
class Token(slexer.Token):
def __repr__(self):
c = self.__class__
module = c.__module__.rsplit('.', 1)[-1]
name = c.__name__
contents = slexer.Token.__repr__(self)
where = self.pos
return '<{0}.{1} {2} at {3}>'.format(module, name, contents, where)
class patternproperty(object):
"""Property that caches the return value of its function and returns that next time.
Use this if the rx attribute (the pattern string to match tokens for) of a Token subclass
is already costly to create and you want it created lazily (i.e. only when parsing starts):
@patternproperty
def rx():
...complicated function returning the regular expression string...
"""
def __init__(self, func):
self.func = func
def __get__(self, instance, owner):
try:
return self.rx
except AttributeError:
self.rx = self.func()
return self.rx
[docs]
class Unparsed(Token):
"""Represents an unparsed piece of input text."""
# some token types with special behaviour:
class Item(Token):
"""A token that decreases the argument count of the current parser."""
def update_state(self, state):
state.endArgument()
class Leaver(Token):
"""A token that leaves the current parser."""
def update_state(self, state):
state.leave()
# some generic types:
[docs]
class Space(Token):
"""A token containing whitespace."""
rx = r'\s+'
[docs]
class Newline(Space):
"""A token that is a single newline."""
rx = r'\n'
# some base types that should be inherited:
[docs]
class String(Token):
"""Base class for tokens that belong to a quote-delimited string."""
[docs]
class StringStart(String):
"""Base class for tokens that start a quote-delimited string."""
[docs]
class StringEnd(String):
"""Base class for tokens that end a quote-delimited string."""
[docs]
class Character(Token):
"""Base class for tokens that are an (escaped) character."""
[docs]
class Numeric(Token):
"""Base class for tokens that are a numerical value."""
[docs]
class Error(Token):
"""Base class for tokens that represent erroneous input."""
# some mixin classes that make special handling of tokens possible besides correct parsing:
# MatchEnd and MatchStart can be used by parenthesis/brace matcher.
# the matchname class attribute defines the type, so that it is independent
# of the way other types could be nested.
[docs]
class MatchStart(object):
"""Mixin class for tokens that have a matching token forward in the text.
The matchname attribute should give a unique name.
"""
matchname = ""
[docs]
class MatchEnd(object):
"""Mixin class for tokens that have a matching token backward in the text.
The matchname attribute should give a unique name.
"""
matchname = ""
# Indent and Dedent can be used by an (auto) indenter.
[docs]
class Indent(object):
"""Mixin class for tokens that have the text on the next line indent more."""
[docs]
class Dedent(object):
"""Mixin class for tokens that have the text on the next line indent less."""