pyparser.py 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. # mako/pyparser.py
  2. # Copyright 2006-2025 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """Handles parsing of Python code.
  7. Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
  8. module is used.
  9. """
  10. import operator
  11. import _ast
  12. from mako import _ast_util
  13. from mako import compat
  14. from mako import exceptions
  15. from mako import util
  16. # words that cannot be assigned to (notably
  17. # smaller than the total keys in __builtins__)
  18. reserved = {"True", "False", "None", "print"}
  19. # the "id" attribute on a function node
  20. arg_id = operator.attrgetter("arg")
  21. util.restore__ast(_ast)
  22. def parse(code, mode="exec", **exception_kwargs):
  23. """Parse an expression into AST"""
  24. try:
  25. return _ast_util.parse(code, "<unknown>", mode)
  26. except Exception as e:
  27. raise exceptions.SyntaxException(
  28. "(%s) %s (%r)"
  29. % (
  30. compat.exception_as().__class__.__name__,
  31. compat.exception_as(),
  32. code[0:50],
  33. ),
  34. **exception_kwargs,
  35. ) from e
  36. class FindIdentifiers(_ast_util.NodeVisitor):
  37. def __init__(self, listener, **exception_kwargs):
  38. self.in_function = False
  39. self.in_assign_targets = False
  40. self.local_ident_stack = set()
  41. self.listener = listener
  42. self.exception_kwargs = exception_kwargs
  43. def _add_declared(self, name):
  44. if not self.in_function:
  45. self.listener.declared_identifiers.add(name)
  46. else:
  47. self.local_ident_stack.add(name)
  48. def visit_ClassDef(self, node):
  49. self._add_declared(node.name)
  50. def visit_Assign(self, node):
  51. # flip around the visiting of Assign so the expression gets
  52. # evaluated first, in the case of a clause like "x=x+5" (x
  53. # is undeclared)
  54. self.visit(node.value)
  55. in_a = self.in_assign_targets
  56. self.in_assign_targets = True
  57. for n in node.targets:
  58. self.visit(n)
  59. self.in_assign_targets = in_a
  60. def visit_ExceptHandler(self, node):
  61. if node.name is not None:
  62. self._add_declared(node.name)
  63. if node.type is not None:
  64. self.visit(node.type)
  65. for statement in node.body:
  66. self.visit(statement)
  67. def visit_Lambda(self, node, *args):
  68. self._visit_function(node, True)
  69. def visit_FunctionDef(self, node):
  70. self._add_declared(node.name)
  71. self._visit_function(node, False)
  72. def visit_ListComp(self, node):
  73. if self.in_function:
  74. for comp in node.generators:
  75. self.visit(comp.target)
  76. self.visit(comp.iter)
  77. else:
  78. self.generic_visit(node)
  79. visit_SetComp = visit_GeneratorExp = visit_ListComp
  80. def visit_DictComp(self, node):
  81. if self.in_function:
  82. for comp in node.generators:
  83. self.visit(comp.target)
  84. self.visit(comp.iter)
  85. else:
  86. self.generic_visit(node)
  87. def _expand_tuples(self, args):
  88. for arg in args:
  89. if isinstance(arg, _ast.Tuple):
  90. yield from arg.elts
  91. else:
  92. yield arg
  93. def _visit_function(self, node, islambda):
  94. # push function state onto stack. dont log any more
  95. # identifiers as "declared" until outside of the function,
  96. # but keep logging identifiers as "undeclared". track
  97. # argument names in each function header so they arent
  98. # counted as "undeclared"
  99. inf = self.in_function
  100. self.in_function = True
  101. local_ident_stack = self.local_ident_stack
  102. self.local_ident_stack = local_ident_stack.union(
  103. [arg_id(arg) for arg in self._expand_tuples(node.args.args)]
  104. )
  105. if islambda:
  106. self.visit(node.body)
  107. else:
  108. for n in node.body:
  109. self.visit(n)
  110. self.in_function = inf
  111. self.local_ident_stack = local_ident_stack
  112. def visit_For(self, node):
  113. # flip around visit
  114. self.visit(node.iter)
  115. self.visit(node.target)
  116. for statement in node.body:
  117. self.visit(statement)
  118. for statement in node.orelse:
  119. self.visit(statement)
  120. def visit_Name(self, node):
  121. if isinstance(node.ctx, _ast.Store):
  122. # this is eqiuvalent to visit_AssName in
  123. # compiler
  124. self._add_declared(node.id)
  125. elif (
  126. node.id not in reserved
  127. and node.id not in self.listener.declared_identifiers
  128. and node.id not in self.local_ident_stack
  129. ):
  130. self.listener.undeclared_identifiers.add(node.id)
  131. def visit_Import(self, node):
  132. for name in node.names:
  133. if name.asname is not None:
  134. self._add_declared(name.asname)
  135. else:
  136. self._add_declared(name.name.split(".")[0])
  137. def visit_ImportFrom(self, node):
  138. for name in node.names:
  139. if name.asname is not None:
  140. self._add_declared(name.asname)
  141. elif name.name == "*":
  142. raise exceptions.CompileException(
  143. "'import *' is not supported, since all identifier "
  144. "names must be explicitly declared. Please use the "
  145. "form 'from <modulename> import <name1>, <name2>, "
  146. "...' instead.",
  147. **self.exception_kwargs,
  148. )
  149. else:
  150. self._add_declared(name.name)
  151. class FindTuple(_ast_util.NodeVisitor):
  152. def __init__(self, listener, code_factory, **exception_kwargs):
  153. self.listener = listener
  154. self.exception_kwargs = exception_kwargs
  155. self.code_factory = code_factory
  156. def visit_Tuple(self, node):
  157. for n in node.elts:
  158. p = self.code_factory(n, **self.exception_kwargs)
  159. self.listener.codeargs.append(p)
  160. self.listener.args.append(ExpressionGenerator(n).value())
  161. ldi = self.listener.declared_identifiers
  162. self.listener.declared_identifiers = ldi.union(
  163. p.declared_identifiers
  164. )
  165. lui = self.listener.undeclared_identifiers
  166. self.listener.undeclared_identifiers = lui.union(
  167. p.undeclared_identifiers
  168. )
  169. class ParseFunc(_ast_util.NodeVisitor):
  170. def __init__(self, listener, **exception_kwargs):
  171. self.listener = listener
  172. self.exception_kwargs = exception_kwargs
  173. def visit_FunctionDef(self, node):
  174. self.listener.funcname = node.name
  175. argnames = [arg_id(arg) for arg in node.args.args]
  176. if node.args.vararg:
  177. argnames.append(node.args.vararg.arg)
  178. kwargnames = [arg_id(arg) for arg in node.args.kwonlyargs]
  179. if node.args.kwarg:
  180. kwargnames.append(node.args.kwarg.arg)
  181. self.listener.argnames = argnames
  182. self.listener.defaults = node.args.defaults # ast
  183. self.listener.kwargnames = kwargnames
  184. self.listener.kwdefaults = node.args.kw_defaults
  185. self.listener.varargs = node.args.vararg
  186. self.listener.kwargs = node.args.kwarg
  187. class ExpressionGenerator:
  188. def __init__(self, astnode):
  189. self.generator = _ast_util.SourceGenerator(" " * 4)
  190. self.generator.visit(astnode)
  191. def value(self):
  192. return "".join(self.generator.result)