Search.setIndex({"docnames": ["index", "lexer", "parser"], "filenames": ["index.rst", "lexer.rst", "parser.rst"], "titles": ["Welcome to ptk\u2019s documentation!", "Lexical analysis", "Syntactic analysis"], "terms": {"content": 0, "lexic": 0, "analysi": 0, "token": [0, 1], "skiptoken": [0, 1], "lexererror": [0, 1], "lexerbas": [0, 1, 2], "relex": [0, 1, 2], "progressivelex": [0, 1], "asynclex": [0, 1, 2], "eof": [0, 1, 2], "syntact": 0, "product": 0, "parseerror": [0, 2], "leftassoc": [0, 2], "rightassoc": [0, 2], "nonassoc": [0, 2], "lrparser": [0, 2], "productionpars": [0, 2], "asynclrpars": [0, 1, 2], "syntax": 0, "wrap": 0, "up": 0, "conflict": 0, "resolut": 0, "rule": 0, "asynchron": [0, 1], "lexer": [0, 1], "parser": 0, "us": [0, 1], "defer": 0, "index": 0, "modul": [0, 2], "search": 0, "page": 0, "regex": 1, "type": [1, 2], "none": [1, 2], "decor": [1, 2], "definit": 1, "class": [1, 2], "deriv": [1, 2], "from": [1, 2], "paramet": [1, 2], "rx": 1, "str": 1, "A": [1, 2], "regular": [1, 2], "express": [1, 2], "defin": 1, "possibl": 1, "valu": 1, "list": [1, 2], "string": 1, "thi": [1, 2], "method": [1, 2], "can": [1, 2], "recogn": [1, 2], "If": 1, "omit": [1, 2], "i": [1, 2], "assum": [1, 2], "": [1, 2], "name": [1, 2], "basic": 1, "usag": 1, "ptk": [1, 2], "import": [1, 2], "mylex": 1, "r": [1, 2], "za": 1, "z_": 1, "z0": 1, "9_": 1, "def": [1, 2], "identifi": [1, 2], "self": [1, 2], "tok": [1, 2], "pass": [1, 2], "an": [1, 2], "which": [1, 2], "The": [1, 2], "hold": [1, 2], "two": 1, "attribut": [1, 2], "you": [1, 2], "modifi": 1, "place": 1, "1": [1, 2], "9": [1, 2], "0": [1, 2], "number": [1, 2], "int": [1, 2], "In": [1, 2], "some": 1, "case": [1, 2], "mai": [1, 2], "necessari": 1, "chang": [1, 2], "well": [1, 2], "instanc": [1, 2], "disambigu": [1, 2], "between": [1, 2], "ar": [1, 2], "builtin": 1, "other": 1, "ones": [1, 2], "order": 1, "know": 1, "gener": [1, 2], "should": [1, 2], "identifier_or_builtin": 1, "len": 1, "id": 1, "els": [1, 2], "default": [1, 2], "must": [1, 2], "set": [1, 2], "let": [1, 2], "caus": 1, "ignor": [1, 2], "depend": [1, 2], "unicod": 1, "byte": [1, 2], "done": 1, "wai": [1, 2], "sever": [1, 2], "match": [1, 2], "input": [1, 2], "longest": 1, "choosen": 1, "equal": 1, "length": 1, "first": [1, 2], "sourc": [1, 2], "code": 1, "declar": [1, 2], "win": 1, "except": [1, 2], "rais": 1, "your": [1, 2], "consum": [1, 2], "char": [1, 2], "po": 1, "unrecogn": 1, "variabl": [1, 2], "lineno": 1, "line": [1, 2], "colno": 1, "column": [1, 2], "interfac": 1, "For": 1, "concret": 1, "implement": 1, "see": [1, 2], "posit": 1, "alia": 1, "field": 1, "2": [1, 2], "return": [1, 2], "current": 1, "stream": [1, 2], "tupl": [1, 2], "advancecolumn": 1, "count": 1, "advanc": 1, "advancelin": 1, "static": [1, 2], "overrid": [1, 2], "charact": [1, 2], "space": [1, 2], "tab": [1, 2], "test": [1, 2], "true": [1, 2], "setconsum": 1, "object": [1, 2], "feed": [1, 2], "all": [1, 2], "seen": [1, 2], "after": [1, 2], "directli": 1, "when": [1, 2], "correspond": 1, "reset": 1, "handi": 1, "pars": [1, 2], "easili": 1, "follow": [1, 2], "c": 1, "without": 1, "have": [1, 2], "neg": [1, 2], "lookahead": 1, "cstring": 1, "__init__": 1, "state": [1, 2], "stringio": 1, "getvalu": 1, "write": 1, "elif": 1, "also": [1, 2], "instead": [1, 2], "comment": 1, "whole": 1, "newtoken": [1, 2], "invok": 1, "soon": 1, "classmethod": 1, "tokentyp": 1, "base": 1, "python": [1, 2], "faster": 1, "than": [1, 2], "onli": [1, 2], "simpl": [1, 2], "pure": 1, "engin": 1, "abl": 1, "progress": 1, "fashion": 1, "just": 1, "call": [1, 2], "whatev": 1, "avail": [1, 2], "thei": 1, "re": 1, "context": 1, "start": [1, 2], "3": [1, 2], "5": [1, 2], "version": [1, 2], "slow": 1, "hell": 1, "handl": 1, "singl": 1, "finish": 1, "argument": [1, 2], "async_lex": [1, 2], "work": [1, 2], "like": [1, 2], "via": 1, "asyncfe": [1, 2], "It": 1, "singleton": 1, "indic": 1, "end": 1, "its": 1, "own": 1, "prod": 2, "prioriti": 2, "grammar": 2, "mypars": 2, "e": 2, "sum": 2, "section": 2, "specifi": 2, "ha": 2, "same": 2, "exist": 2, "typic": 2, "unari": 2, "minu": 2, "binari": 2, "been": 2, "long": 2, "explicit": 2, "one": 2, "associ": 2, "unaryminu": 2, "non": 2, "higher": 2, "anyth": 2, "error": 2, "unexpect": 2, "expect": 2, "would": 2, "valid": 2, "wa": 2, "encount": 2, "lasttoken": 2, "last": 2, "befor": 2, "left": 2, "later": 2, "so": 2, "right": 2, "remark": 2, "lr": 2, "intend": 2, "inherit": 2, "newsent": 2, "sentenc": 2, "symbol": 2, "reduc": 2, "callback": 2, "grammarclass": 2, "startsymbol": 2, "async_pars": 2, "support": 2, "new": 2, "conjuct": 2, "And": 2, "action": 2, "asyncnewsent": 2, "await": 2, "through": 2, "variant": 2, "bnf": 2, "exampl": 2, "plu": 2, "binaryop_sum": 2, "binaryop_minu": 2, "here": 2, "termin": 2, "uppercas": 2, "lowercas": 2, "convent": 2, "don": 2, "t": 2, "need": 2, "separ": 2, "group": 2, "either": 2, "binaryop": 2, "Or": 2, "time": 2, "item": 2, "side": 2, "specif": 2, "each": 2, "keyword": 2, "bracket": 2, "thu": 2, "altern": 2, "slightli": 2, "behavior": 2, "actual": 2, "symnam": 2, "left_bracket": 2, "right_bracket": 2, "form": 2, "second": 2, "want": 2, "store": 2, "warn": 2, "report": 2, "part": 2, "get": 2, "inform": 2, "those": 2, "appear": 2, "doubl": 2, "quot": 2, "allow": 2, "skip": 2, "level": 2, "consid": 2, "becaus": 2, "note": 2, "nontermin": 2, "immedi": 2, "among": 2, "mean": 2, "sugar": 2, "under": 2, "hood": 2, "addit": 2, "b": 2, "equival": 2, "empti": 2, "appli": 2, "l_b": 2, "cannot": 2, "addition": 2, "includ": 2, "paren": 2, "still": 2, "fulli": 2, "function": 2, "four": 2, "integ": 2, "calcul": 2, "n": 2, "val": 2, "op": 2, "add": 2, "sub": 2, "mul": 2, "floordiv": 2, "comma": 2, "integer_list": 2, "print": 2, "yacc": 2, "bison": 2, "shift": 2, "resolv": 2, "choos": 2, "reduct": 2, "sinc": 2, "introduc": 2, "instal": 2, "async_gener": 2, "idea": 2, "unambigu": 2, "turn": 2, "asyncpars": 2, "asyncnewtoken": 2, "result": 2, "sampl": 2, "directori": 2, "contain": 2, "usr": 2, "bin": 2, "env": 2, "utf": 2, "8": 2, "due": 2, "variou": 2, "buffer": 2, "problem": 2, "probabl": 2, "won": 2, "what": 2, "point": 2, "unless": 2, "forc": 2, "stdin": 2, "noninteract": 2, "g": 2, "echo": 2, "4": 2, "6": 2, "python3": 2, "async_calc": 2, "py": 2, "o": 2, "asyncio": 2, "sy": 2, "codec": 2, "async": 2, "d": 2, "main": 2, "reader": 2, "streamread": 2, "get_event_loop": 2, "connect_read_pip": 2, "lambda": 2, "streamreaderprotocol": 2, "decod": 2, "getincrementaldecod": 2, "utf_8": 2, "while": 2, "read": 2, "break": 2, "repr": 2, "loop": 2, "run_until_complet": 2, "deferredlex": 2, "deferredlrpars": 2, "twist": 2, "special": 2, "defernewtoken": 2, "func": 2, "defernewsent": 2, "defer_calc": 2, "detail": 2}, "objects": {"": [[2, 0, 1, "", "production"], [1, 0, 1, "", "token"]], "ptk.async_lexer": [[1, 1, 1, "", "AsyncLexer"]], "ptk.async_parser": [[2, 1, 1, "", "AsyncLRParser"]], "ptk": [[1, 2, 0, "-", "lexer"], [2, 2, 0, "-", "parser"]], "ptk.lexer": [[1, 3, 1, "", "EOF"], [1, 1, 1, "", "LexerBase"], [1, 6, 1, "", "LexerError"], [1, 1, 1, "", "ProgressiveLexer"], [1, 1, 1, "", "ReLexer"], [1, 6, 1, "", "SkipToken"]], "ptk.lexer.LexerBase": [[1, 1, 1, "", "Token"], [1, 5, 1, "", "advanceColumn"], [1, 5, 1, "", "advanceLine"], [1, 5, 1, "", "ignore"], [1, 5, 1, "", "newToken"], [1, 5, 1, "", "parse"], [1, 5, 1, "", "position"], [1, 5, 1, "", "setConsumer"], [1, 5, 1, "", "tokenTypes"]], "ptk.lexer.LexerBase.Token": [[1, 4, 1, "", "position"], [1, 4, 1, "", "type"], [1, 4, 1, "", "value"]], "ptk.lexer.ProgressiveLexer": [[1, 5, 1, "", "feed"], [1, 5, 1, "", "parse"]], "ptk.lexer.ReLexer": [[1, 5, 1, "", "parse"]], "ptk.parser": [[2, 1, 1, "", "LRParser"], [2, 6, 1, "", "ParseError"], [2, 1, 1, "", "ProductionParser"], [2, 0, 1, "", "leftAssoc"], [2, 0, 1, "", "nonAssoc"], [2, 0, 1, "", "rightAssoc"]], "ptk.parser.LRParser": [[2, 5, 1, "", "newSentence"]], "ptk.parser.ParseError": [[2, 5, 1, "", "expecting"], [2, 5, 1, "", "lastToken"], [2, 5, 1, "", "state"], [2, 5, 1, "", "tokens"]], "ptk.parser.ProductionParser": [[2, 5, 1, "", "ignore"], [2, 5, 1, "", "newSentence"]]}, "objtypes": {"0": "py:function", "1": "py:class", "2": "py:module", "3": "py:data", "4": "py:attribute", "5": "py:method", "6": "py:exception"}, "objnames": {"0": ["py", "function", "Python function"], "1": ["py", "class", "Python class"], "2": ["py", "module", "Python module"], "3": ["py", "data", "Python data"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "method", "Python method"], "6": ["py", "exception", "Python exception"]}, "titleterms": {"welcom": 0, "ptk": 0, "": 0, "document": 0, "indic": 0, "tabl": 0, "lexic": 1, "analysi": [1, 2], "syntact": 2, "product": 2, "syntax": 2, "basic": 2, "semant": 2, "valu": 2, "posit": 2, "code": 2, "litter": 2, "token": 2, "repeat": 2, "oper": 2, "wrap": 2, "up": 2, "conflict": 2, "resolut": 2, "rule": 2, "asynchron": 2, "lexer": 2, "parser": 2, "us": 2, "defer": 2}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Welcome to ptk\u2019s documentation!": [[0, "welcome-to-ptk-s-documentation"]], "Indices and tables": [[0, "indices-and-tables"]], "Lexical analysis": [[1, "lexical-analysis"]], "Syntactic analysis": [[2, "syntactic-analysis"]], "Production syntax": [[2, "production-syntax"]], "Basics": [[2, "basics"]], "Semantic values": [[2, "semantic-values"]], "Position in code": [[2, "position-in-code"]], "Litteral tokens": [[2, "litteral-tokens"]], "Repeat operators": [[2, "repeat-operators"]], "Wrapping it up": [[2, "wrapping-it-up"]], "Conflict resolution rules": [[2, "conflict-resolution-rules"]], "Asynchronous lexer/parser": [[2, "asynchronous-lexer-parser"]], "Asynchronous lexer/parser using Deferreds": [[2, "asynchronous-lexer-parser-using-deferreds"]]}, "indexentries": {"asynclexer (class in ptk.async_lexer)": [[1, "ptk.async_lexer.AsyncLexer"]], "eof (in module ptk.lexer)": [[1, "ptk.lexer.EOF"]], "lexerbase (class in ptk.lexer)": [[1, "ptk.lexer.LexerBase"]], "lexerbase.token (class in ptk.lexer)": [[1, "ptk.lexer.LexerBase.Token"]], "lexererror": [[1, "ptk.lexer.LexerError"]], "progressivelexer (class in ptk.lexer)": [[1, "ptk.lexer.ProgressiveLexer"]], "relexer (class in ptk.lexer)": [[1, "ptk.lexer.ReLexer"]], "skiptoken": [[1, "ptk.lexer.SkipToken"]], "advancecolumn() (ptk.lexer.lexerbase method)": [[1, "ptk.lexer.LexerBase.advanceColumn"]], "advanceline() (ptk.lexer.lexerbase method)": [[1, "ptk.lexer.LexerBase.advanceLine"]], "built-in function": [[1, "token"], [2, "production"]], "feed() (ptk.lexer.progressivelexer method)": [[1, "ptk.lexer.ProgressiveLexer.feed"]], "ignore() (ptk.lexer.lexerbase static method)": [[1, "ptk.lexer.LexerBase.ignore"]], "module": [[1, "module-ptk.lexer"], [2, "module-ptk.parser"]], "newtoken() (ptk.lexer.lexerbase method)": [[1, "ptk.lexer.LexerBase.newToken"]], "parse() (ptk.lexer.lexerbase method)": [[1, "ptk.lexer.LexerBase.parse"]], "parse() (ptk.lexer.progressivelexer method)": [[1, "ptk.lexer.ProgressiveLexer.parse"]], "parse() (ptk.lexer.relexer method)": [[1, "ptk.lexer.ReLexer.parse"]], "position (ptk.lexer.lexerbase.token attribute)": [[1, "ptk.lexer.LexerBase.Token.position"]], "position() (ptk.lexer.lexerbase method)": [[1, "ptk.lexer.LexerBase.position"]], "ptk.lexer": [[1, "module-ptk.lexer"]], "setconsumer() (ptk.lexer.lexerbase method)": [[1, "ptk.lexer.LexerBase.setConsumer"]], "token()": [[1, "token"]], "tokentypes() (ptk.lexer.lexerbase class method)": [[1, "ptk.lexer.LexerBase.tokenTypes"]], "type (ptk.lexer.lexerbase.token attribute)": [[1, "ptk.lexer.LexerBase.Token.type"]], "value (ptk.lexer.lexerbase.token attribute)": [[1, "ptk.lexer.LexerBase.Token.value"]], "asynclrparser (class in ptk.async_parser)": [[2, "ptk.async_parser.AsyncLRParser"]], "lrparser (class in ptk.parser)": [[2, "ptk.parser.LRParser"]], "parseerror": [[2, "ptk.parser.ParseError"]], "productionparser (class in ptk.parser)": [[2, "ptk.parser.ProductionParser"]], "expecting() (ptk.parser.parseerror method)": [[2, "ptk.parser.ParseError.expecting"]], "ignore() (ptk.parser.productionparser static method)": [[2, "ptk.parser.ProductionParser.ignore"]], "lasttoken() (ptk.parser.parseerror method)": [[2, "ptk.parser.ParseError.lastToken"]], "leftassoc() (in module ptk.parser)": [[2, "ptk.parser.leftAssoc"]], "newsentence() (ptk.parser.lrparser method)": [[2, "ptk.parser.LRParser.newSentence"]], "newsentence() (ptk.parser.productionparser method)": [[2, "ptk.parser.ProductionParser.newSentence"]], "nonassoc() (in module ptk.parser)": [[2, "ptk.parser.nonAssoc"]], "production()": [[2, "production"]], "ptk.parser": [[2, "module-ptk.parser"]], "rightassoc() (in module ptk.parser)": [[2, "ptk.parser.rightAssoc"]], "state() (ptk.parser.parseerror method)": [[2, "ptk.parser.ParseError.state"]], "tokens() (ptk.parser.parseerror method)": [[2, "ptk.parser.ParseError.tokens"]]}})