_ast_util.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713
  1. # mako/_ast_util.py
  2. # Copyright 2006-2025 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """
  7. ast
  8. ~~~
  9. This is a stripped down version of Armin Ronacher's ast module.
  10. :copyright: Copyright 2008 by Armin Ronacher.
  11. :license: Python License.
  12. """
  13. from _ast import Add
  14. from _ast import And
  15. from _ast import AST
  16. from _ast import BitAnd
  17. from _ast import BitOr
  18. from _ast import BitXor
  19. from _ast import Div
  20. from _ast import Eq
  21. from _ast import FloorDiv
  22. from _ast import Gt
  23. from _ast import GtE
  24. from _ast import If
  25. from _ast import In
  26. from _ast import Invert
  27. from _ast import Is
  28. from _ast import IsNot
  29. from _ast import LShift
  30. from _ast import Lt
  31. from _ast import LtE
  32. from _ast import Mod
  33. from _ast import Mult
  34. from _ast import Name
  35. from _ast import Not
  36. from _ast import NotEq
  37. from _ast import NotIn
  38. from _ast import Or
  39. from _ast import PyCF_ONLY_AST
  40. from _ast import RShift
  41. from _ast import Sub
  42. from _ast import UAdd
  43. from _ast import USub
  44. BOOLOP_SYMBOLS = {And: "and", Or: "or"}
  45. BINOP_SYMBOLS = {
  46. Add: "+",
  47. Sub: "-",
  48. Mult: "*",
  49. Div: "/",
  50. FloorDiv: "//",
  51. Mod: "%",
  52. LShift: "<<",
  53. RShift: ">>",
  54. BitOr: "|",
  55. BitAnd: "&",
  56. BitXor: "^",
  57. }
  58. CMPOP_SYMBOLS = {
  59. Eq: "==",
  60. Gt: ">",
  61. GtE: ">=",
  62. In: "in",
  63. Is: "is",
  64. IsNot: "is not",
  65. Lt: "<",
  66. LtE: "<=",
  67. NotEq: "!=",
  68. NotIn: "not in",
  69. }
  70. UNARYOP_SYMBOLS = {Invert: "~", Not: "not", UAdd: "+", USub: "-"}
  71. ALL_SYMBOLS = {}
  72. ALL_SYMBOLS.update(BOOLOP_SYMBOLS)
  73. ALL_SYMBOLS.update(BINOP_SYMBOLS)
  74. ALL_SYMBOLS.update(CMPOP_SYMBOLS)
  75. ALL_SYMBOLS.update(UNARYOP_SYMBOLS)
  76. def parse(expr, filename="<unknown>", mode="exec"):
  77. """Parse an expression into an AST node."""
  78. return compile(expr, filename, mode, PyCF_ONLY_AST)
  79. def iter_fields(node):
  80. """Iterate over all fields of a node, only yielding existing fields."""
  81. for field in node._fields:
  82. try:
  83. yield field, getattr(node, field)
  84. except AttributeError:
  85. pass
  86. class NodeVisitor:
  87. """
  88. Walks the abstract syntax tree and call visitor functions for every node
  89. found. The visitor functions may return values which will be forwarded
  90. by the `visit` method.
  91. Per default the visitor functions for the nodes are ``'visit_'`` +
  92. class name of the node. So a `TryFinally` node visit function would
  93. be `visit_TryFinally`. This behavior can be changed by overriding
  94. the `get_visitor` function. If no visitor function exists for a node
  95. (return value `None`) the `generic_visit` visitor is used instead.
  96. Don't use the `NodeVisitor` if you want to apply changes to nodes during
  97. traversing. For this a special visitor exists (`NodeTransformer`) that
  98. allows modifications.
  99. """
  100. def get_visitor(self, node):
  101. """
  102. Return the visitor function for this node or `None` if no visitor
  103. exists for this node. In that case the generic visit function is
  104. used instead.
  105. """
  106. method = "visit_" + node.__class__.__name__
  107. return getattr(self, method, None)
  108. def visit(self, node):
  109. """Visit a node."""
  110. f = self.get_visitor(node)
  111. if f is not None:
  112. return f(node)
  113. return self.generic_visit(node)
  114. def generic_visit(self, node):
  115. """Called if no explicit visitor function exists for a node."""
  116. for field, value in iter_fields(node):
  117. if isinstance(value, list):
  118. for item in value:
  119. if isinstance(item, AST):
  120. self.visit(item)
  121. elif isinstance(value, AST):
  122. self.visit(value)
  123. class NodeTransformer(NodeVisitor):
  124. """
  125. Walks the abstract syntax tree and allows modifications of nodes.
  126. The `NodeTransformer` will walk the AST and use the return value of the
  127. visitor functions to replace or remove the old node. If the return
  128. value of the visitor function is `None` the node will be removed
  129. from the previous location otherwise it's replaced with the return
  130. value. The return value may be the original node in which case no
  131. replacement takes place.
  132. Here an example transformer that rewrites all `foo` to `data['foo']`::
  133. class RewriteName(NodeTransformer):
  134. def visit_Name(self, node):
  135. return copy_location(Subscript(
  136. value=Name(id='data', ctx=Load()),
  137. slice=Index(value=Str(s=node.id)),
  138. ctx=node.ctx
  139. ), node)
  140. Keep in mind that if the node you're operating on has child nodes
  141. you must either transform the child nodes yourself or call the generic
  142. visit function for the node first.
  143. Nodes that were part of a collection of statements (that applies to
  144. all statement nodes) may also return a list of nodes rather than just
  145. a single node.
  146. Usually you use the transformer like this::
  147. node = YourTransformer().visit(node)
  148. """
  149. def generic_visit(self, node):
  150. for field, old_value in iter_fields(node):
  151. old_value = getattr(node, field, None)
  152. if isinstance(old_value, list):
  153. new_values = []
  154. for value in old_value:
  155. if isinstance(value, AST):
  156. value = self.visit(value)
  157. if value is None:
  158. continue
  159. elif not isinstance(value, AST):
  160. new_values.extend(value)
  161. continue
  162. new_values.append(value)
  163. old_value[:] = new_values
  164. elif isinstance(old_value, AST):
  165. new_node = self.visit(old_value)
  166. if new_node is None:
  167. delattr(node, field)
  168. else:
  169. setattr(node, field, new_node)
  170. return node
  171. class SourceGenerator(NodeVisitor):
  172. """
  173. This visitor is able to transform a well formed syntax tree into python
  174. sourcecode. For more details have a look at the docstring of the
  175. `node_to_source` function.
  176. """
  177. def __init__(self, indent_with):
  178. self.result = []
  179. self.indent_with = indent_with
  180. self.indentation = 0
  181. self.new_lines = 0
  182. def write(self, x):
  183. if self.new_lines:
  184. if self.result:
  185. self.result.append("\n" * self.new_lines)
  186. self.result.append(self.indent_with * self.indentation)
  187. self.new_lines = 0
  188. self.result.append(x)
  189. def newline(self, n=1):
  190. self.new_lines = max(self.new_lines, n)
  191. def body(self, statements):
  192. self.new_line = True
  193. self.indentation += 1
  194. for stmt in statements:
  195. self.visit(stmt)
  196. self.indentation -= 1
  197. def body_or_else(self, node):
  198. self.body(node.body)
  199. if node.orelse:
  200. self.newline()
  201. self.write("else:")
  202. self.body(node.orelse)
  203. def signature(self, node):
  204. want_comma = []
  205. def write_comma():
  206. if want_comma:
  207. self.write(", ")
  208. else:
  209. want_comma.append(True)
  210. padding = [None] * (len(node.args) - len(node.defaults))
  211. for arg, default in zip(node.args, padding + node.defaults):
  212. write_comma()
  213. self.visit(arg)
  214. if default is not None:
  215. self.write("=")
  216. self.visit(default)
  217. if node.vararg is not None:
  218. write_comma()
  219. self.write("*" + node.vararg.arg)
  220. if node.kwarg is not None:
  221. write_comma()
  222. self.write("**" + node.kwarg.arg)
  223. def decorators(self, node):
  224. for decorator in node.decorator_list:
  225. self.newline()
  226. self.write("@")
  227. self.visit(decorator)
  228. # Statements
  229. def visit_Assign(self, node):
  230. self.newline()
  231. for idx, target in enumerate(node.targets):
  232. if idx:
  233. self.write(", ")
  234. self.visit(target)
  235. self.write(" = ")
  236. self.visit(node.value)
  237. def visit_AugAssign(self, node):
  238. self.newline()
  239. self.visit(node.target)
  240. self.write(BINOP_SYMBOLS[type(node.op)] + "=")
  241. self.visit(node.value)
  242. def visit_ImportFrom(self, node):
  243. self.newline()
  244. self.write("from %s%s import " % ("." * node.level, node.module))
  245. for idx, item in enumerate(node.names):
  246. if idx:
  247. self.write(", ")
  248. self.write(item)
  249. def visit_Import(self, node):
  250. self.newline()
  251. for item in node.names:
  252. self.write("import ")
  253. self.visit(item)
  254. def visit_Expr(self, node):
  255. self.newline()
  256. self.generic_visit(node)
  257. def visit_FunctionDef(self, node):
  258. self.newline(n=2)
  259. self.decorators(node)
  260. self.newline()
  261. self.write("def %s(" % node.name)
  262. self.signature(node.args)
  263. self.write("):")
  264. self.body(node.body)
  265. def visit_ClassDef(self, node):
  266. have_args = []
  267. def paren_or_comma():
  268. if have_args:
  269. self.write(", ")
  270. else:
  271. have_args.append(True)
  272. self.write("(")
  273. self.newline(n=3)
  274. self.decorators(node)
  275. self.newline()
  276. self.write("class %s" % node.name)
  277. for base in node.bases:
  278. paren_or_comma()
  279. self.visit(base)
  280. # XXX: the if here is used to keep this module compatible
  281. # with python 2.6.
  282. if hasattr(node, "keywords"):
  283. for keyword in node.keywords:
  284. paren_or_comma()
  285. self.write(keyword.arg + "=")
  286. self.visit(keyword.value)
  287. if getattr(node, "starargs", None):
  288. paren_or_comma()
  289. self.write("*")
  290. self.visit(node.starargs)
  291. if getattr(node, "kwargs", None):
  292. paren_or_comma()
  293. self.write("**")
  294. self.visit(node.kwargs)
  295. self.write(have_args and "):" or ":")
  296. self.body(node.body)
  297. def visit_If(self, node):
  298. self.newline()
  299. self.write("if ")
  300. self.visit(node.test)
  301. self.write(":")
  302. self.body(node.body)
  303. while True:
  304. else_ = node.orelse
  305. if len(else_) == 1 and isinstance(else_[0], If):
  306. node = else_[0]
  307. self.newline()
  308. self.write("elif ")
  309. self.visit(node.test)
  310. self.write(":")
  311. self.body(node.body)
  312. else:
  313. self.newline()
  314. self.write("else:")
  315. self.body(else_)
  316. break
  317. def visit_For(self, node):
  318. self.newline()
  319. self.write("for ")
  320. self.visit(node.target)
  321. self.write(" in ")
  322. self.visit(node.iter)
  323. self.write(":")
  324. self.body_or_else(node)
  325. def visit_While(self, node):
  326. self.newline()
  327. self.write("while ")
  328. self.visit(node.test)
  329. self.write(":")
  330. self.body_or_else(node)
  331. def visit_With(self, node):
  332. self.newline()
  333. self.write("with ")
  334. self.visit(node.context_expr)
  335. if node.optional_vars is not None:
  336. self.write(" as ")
  337. self.visit(node.optional_vars)
  338. self.write(":")
  339. self.body(node.body)
  340. def visit_Pass(self, node):
  341. self.newline()
  342. self.write("pass")
  343. def visit_Print(self, node):
  344. # XXX: python 2.6 only
  345. self.newline()
  346. self.write("print ")
  347. want_comma = False
  348. if node.dest is not None:
  349. self.write(" >> ")
  350. self.visit(node.dest)
  351. want_comma = True
  352. for value in node.values:
  353. if want_comma:
  354. self.write(", ")
  355. self.visit(value)
  356. want_comma = True
  357. if not node.nl:
  358. self.write(",")
  359. def visit_Delete(self, node):
  360. self.newline()
  361. self.write("del ")
  362. for idx, target in enumerate(node):
  363. if idx:
  364. self.write(", ")
  365. self.visit(target)
  366. def visit_TryExcept(self, node):
  367. self.newline()
  368. self.write("try:")
  369. self.body(node.body)
  370. for handler in node.handlers:
  371. self.visit(handler)
  372. def visit_TryFinally(self, node):
  373. self.newline()
  374. self.write("try:")
  375. self.body(node.body)
  376. self.newline()
  377. self.write("finally:")
  378. self.body(node.finalbody)
  379. def visit_Global(self, node):
  380. self.newline()
  381. self.write("global " + ", ".join(node.names))
  382. def visit_Nonlocal(self, node):
  383. self.newline()
  384. self.write("nonlocal " + ", ".join(node.names))
  385. def visit_Return(self, node):
  386. self.newline()
  387. self.write("return ")
  388. self.visit(node.value)
  389. def visit_Break(self, node):
  390. self.newline()
  391. self.write("break")
  392. def visit_Continue(self, node):
  393. self.newline()
  394. self.write("continue")
  395. def visit_Raise(self, node):
  396. # XXX: Python 2.6 / 3.0 compatibility
  397. self.newline()
  398. self.write("raise")
  399. if hasattr(node, "exc") and node.exc is not None:
  400. self.write(" ")
  401. self.visit(node.exc)
  402. if node.cause is not None:
  403. self.write(" from ")
  404. self.visit(node.cause)
  405. elif hasattr(node, "type") and node.type is not None:
  406. self.visit(node.type)
  407. if node.inst is not None:
  408. self.write(", ")
  409. self.visit(node.inst)
  410. if node.tback is not None:
  411. self.write(", ")
  412. self.visit(node.tback)
  413. # Expressions
  414. def visit_Attribute(self, node):
  415. self.visit(node.value)
  416. self.write("." + node.attr)
  417. def visit_Call(self, node):
  418. want_comma = []
  419. def write_comma():
  420. if want_comma:
  421. self.write(", ")
  422. else:
  423. want_comma.append(True)
  424. self.visit(node.func)
  425. self.write("(")
  426. for arg in node.args:
  427. write_comma()
  428. self.visit(arg)
  429. for keyword in node.keywords:
  430. write_comma()
  431. self.write(keyword.arg + "=")
  432. self.visit(keyword.value)
  433. if getattr(node, "starargs", None):
  434. write_comma()
  435. self.write("*")
  436. self.visit(node.starargs)
  437. if getattr(node, "kwargs", None):
  438. write_comma()
  439. self.write("**")
  440. self.visit(node.kwargs)
  441. self.write(")")
  442. def visit_Name(self, node):
  443. self.write(node.id)
  444. def visit_NameConstant(self, node):
  445. self.write(str(node.value))
  446. def visit_arg(self, node):
  447. self.write(node.arg)
  448. def visit_Str(self, node):
  449. self.write(repr(node.s))
  450. def visit_Bytes(self, node):
  451. self.write(repr(node.s))
  452. def visit_Num(self, node):
  453. self.write(repr(node.n))
  454. # newly needed in Python 3.8
  455. def visit_Constant(self, node):
  456. self.write(repr(node.value))
  457. def visit_Tuple(self, node):
  458. self.write("(")
  459. idx = -1
  460. for idx, item in enumerate(node.elts):
  461. if idx:
  462. self.write(", ")
  463. self.visit(item)
  464. self.write(idx and ")" or ",)")
  465. def sequence_visit(left, right):
  466. def visit(self, node):
  467. self.write(left)
  468. for idx, item in enumerate(node.elts):
  469. if idx:
  470. self.write(", ")
  471. self.visit(item)
  472. self.write(right)
  473. return visit
  474. visit_List = sequence_visit("[", "]")
  475. visit_Set = sequence_visit("{", "}")
  476. del sequence_visit
  477. def visit_Dict(self, node):
  478. self.write("{")
  479. for idx, (key, value) in enumerate(zip(node.keys, node.values)):
  480. if idx:
  481. self.write(", ")
  482. self.visit(key)
  483. self.write(": ")
  484. self.visit(value)
  485. self.write("}")
  486. def visit_BinOp(self, node):
  487. self.write("(")
  488. self.visit(node.left)
  489. self.write(" %s " % BINOP_SYMBOLS[type(node.op)])
  490. self.visit(node.right)
  491. self.write(")")
  492. def visit_BoolOp(self, node):
  493. self.write("(")
  494. for idx, value in enumerate(node.values):
  495. if idx:
  496. self.write(" %s " % BOOLOP_SYMBOLS[type(node.op)])
  497. self.visit(value)
  498. self.write(")")
  499. def visit_Compare(self, node):
  500. self.write("(")
  501. self.visit(node.left)
  502. for op, right in zip(node.ops, node.comparators):
  503. self.write(" %s " % CMPOP_SYMBOLS[type(op)])
  504. self.visit(right)
  505. self.write(")")
  506. def visit_UnaryOp(self, node):
  507. self.write("(")
  508. op = UNARYOP_SYMBOLS[type(node.op)]
  509. self.write(op)
  510. if op == "not":
  511. self.write(" ")
  512. self.visit(node.operand)
  513. self.write(")")
  514. def visit_Subscript(self, node):
  515. self.visit(node.value)
  516. self.write("[")
  517. self.visit(node.slice)
  518. self.write("]")
  519. def visit_Slice(self, node):
  520. if node.lower is not None:
  521. self.visit(node.lower)
  522. self.write(":")
  523. if node.upper is not None:
  524. self.visit(node.upper)
  525. if node.step is not None:
  526. self.write(":")
  527. if not (isinstance(node.step, Name) and node.step.id == "None"):
  528. self.visit(node.step)
  529. def visit_ExtSlice(self, node):
  530. for idx, item in node.dims:
  531. if idx:
  532. self.write(", ")
  533. self.visit(item)
  534. def visit_Yield(self, node):
  535. self.write("yield ")
  536. self.visit(node.value)
  537. def visit_Lambda(self, node):
  538. self.write("lambda ")
  539. self.signature(node.args)
  540. self.write(": ")
  541. self.visit(node.body)
  542. def visit_Ellipsis(self, node):
  543. self.write("Ellipsis")
  544. def generator_visit(left, right):
  545. def visit(self, node):
  546. self.write(left)
  547. self.visit(node.elt)
  548. for comprehension in node.generators:
  549. self.visit(comprehension)
  550. self.write(right)
  551. return visit
  552. visit_ListComp = generator_visit("[", "]")
  553. visit_GeneratorExp = generator_visit("(", ")")
  554. visit_SetComp = generator_visit("{", "}")
  555. del generator_visit
  556. def visit_DictComp(self, node):
  557. self.write("{")
  558. self.visit(node.key)
  559. self.write(": ")
  560. self.visit(node.value)
  561. for comprehension in node.generators:
  562. self.visit(comprehension)
  563. self.write("}")
  564. def visit_IfExp(self, node):
  565. self.visit(node.body)
  566. self.write(" if ")
  567. self.visit(node.test)
  568. self.write(" else ")
  569. self.visit(node.orelse)
  570. def visit_Starred(self, node):
  571. self.write("*")
  572. self.visit(node.value)
  573. def visit_Repr(self, node):
  574. # XXX: python 2.6 only
  575. self.write("`")
  576. self.visit(node.value)
  577. self.write("`")
  578. # Helper Nodes
  579. def visit_alias(self, node):
  580. self.write(node.name)
  581. if node.asname is not None:
  582. self.write(" as " + node.asname)
  583. def visit_comprehension(self, node):
  584. self.write(" for ")
  585. self.visit(node.target)
  586. self.write(" in ")
  587. self.visit(node.iter)
  588. if node.ifs:
  589. for if_ in node.ifs:
  590. self.write(" if ")
  591. self.visit(if_)
  592. def visit_excepthandler(self, node):
  593. self.newline()
  594. self.write("except")
  595. if node.type is not None:
  596. self.write(" ")
  597. self.visit(node.type)
  598. if node.name is not None:
  599. self.write(" as ")
  600. self.visit(node.name)
  601. self.write(":")
  602. self.body(node.body)