parsetree.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656
  1. # mako/parsetree.py
  2. # Copyright 2006-2025 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """defines the parse tree components for Mako templates."""
  7. import re
  8. from mako import ast
  9. from mako import exceptions
  10. from mako import filters
  11. from mako import util
  12. class Node:
  13. """base class for a Node in the parse tree."""
  14. def __init__(self, source, lineno, pos, filename):
  15. self.source = source
  16. self.lineno = lineno
  17. self.pos = pos
  18. self.filename = filename
  19. @property
  20. def exception_kwargs(self):
  21. return {
  22. "source": self.source,
  23. "lineno": self.lineno,
  24. "pos": self.pos,
  25. "filename": self.filename,
  26. }
  27. def get_children(self):
  28. return []
  29. def accept_visitor(self, visitor):
  30. def traverse(node):
  31. for n in node.get_children():
  32. n.accept_visitor(visitor)
  33. method = getattr(visitor, "visit" + self.__class__.__name__, traverse)
  34. method(self)
  35. class TemplateNode(Node):
  36. """a 'container' node that stores the overall collection of nodes."""
  37. def __init__(self, filename):
  38. super().__init__("", 0, 0, filename)
  39. self.nodes = []
  40. self.page_attributes = {}
  41. def get_children(self):
  42. return self.nodes
  43. def __repr__(self):
  44. return "TemplateNode(%s, %r)" % (
  45. util.sorted_dict_repr(self.page_attributes),
  46. self.nodes,
  47. )
  48. class ControlLine(Node):
  49. """defines a control line, a line-oriented python line or end tag.
  50. e.g.::
  51. % if foo:
  52. (markup)
  53. % endif
  54. """
  55. has_loop_context = False
  56. def __init__(self, keyword, isend, text, **kwargs):
  57. super().__init__(**kwargs)
  58. self.text = text
  59. self.keyword = keyword
  60. self.isend = isend
  61. self.is_primary = keyword in ["for", "if", "while", "try", "with"]
  62. self.nodes = []
  63. if self.isend:
  64. self._declared_identifiers = []
  65. self._undeclared_identifiers = []
  66. else:
  67. code = ast.PythonFragment(text, **self.exception_kwargs)
  68. self._declared_identifiers = code.declared_identifiers
  69. self._undeclared_identifiers = code.undeclared_identifiers
  70. def get_children(self):
  71. return self.nodes
  72. def declared_identifiers(self):
  73. return self._declared_identifiers
  74. def undeclared_identifiers(self):
  75. return self._undeclared_identifiers
  76. def is_ternary(self, keyword):
  77. """return true if the given keyword is a ternary keyword
  78. for this ControlLine"""
  79. cases = {
  80. "if": {"else", "elif"},
  81. "try": {"except", "finally"},
  82. "for": {"else"},
  83. }
  84. return keyword in cases.get(self.keyword, set())
  85. def __repr__(self):
  86. return "ControlLine(%r, %r, %r, %r)" % (
  87. self.keyword,
  88. self.text,
  89. self.isend,
  90. (self.lineno, self.pos),
  91. )
  92. class Text(Node):
  93. """defines plain text in the template."""
  94. def __init__(self, content, **kwargs):
  95. super().__init__(**kwargs)
  96. self.content = content
  97. def __repr__(self):
  98. return "Text(%r, %r)" % (self.content, (self.lineno, self.pos))
  99. class Code(Node):
  100. """defines a Python code block, either inline or module level.
  101. e.g.::
  102. inline:
  103. <%
  104. x = 12
  105. %>
  106. module level:
  107. <%!
  108. import logger
  109. %>
  110. """
  111. def __init__(self, text, ismodule, **kwargs):
  112. super().__init__(**kwargs)
  113. self.text = text
  114. self.ismodule = ismodule
  115. self.code = ast.PythonCode(text, **self.exception_kwargs)
  116. def declared_identifiers(self):
  117. return self.code.declared_identifiers
  118. def undeclared_identifiers(self):
  119. return self.code.undeclared_identifiers
  120. def __repr__(self):
  121. return "Code(%r, %r, %r)" % (
  122. self.text,
  123. self.ismodule,
  124. (self.lineno, self.pos),
  125. )
  126. class Comment(Node):
  127. """defines a comment line.
  128. # this is a comment
  129. """
  130. def __init__(self, text, **kwargs):
  131. super().__init__(**kwargs)
  132. self.text = text
  133. def __repr__(self):
  134. return "Comment(%r, %r)" % (self.text, (self.lineno, self.pos))
  135. class Expression(Node):
  136. """defines an inline expression.
  137. ${x+y}
  138. """
  139. def __init__(self, text, escapes, **kwargs):
  140. super().__init__(**kwargs)
  141. self.text = text
  142. self.escapes = escapes
  143. self.escapes_code = ast.ArgumentList(escapes, **self.exception_kwargs)
  144. self.code = ast.PythonCode(text, **self.exception_kwargs)
  145. def declared_identifiers(self):
  146. return []
  147. def undeclared_identifiers(self):
  148. # TODO: make the "filter" shortcut list configurable at parse/gen time
  149. return self.code.undeclared_identifiers.union(
  150. self.escapes_code.undeclared_identifiers.difference(
  151. filters.DEFAULT_ESCAPES
  152. )
  153. ).difference(self.code.declared_identifiers)
  154. def __repr__(self):
  155. return "Expression(%r, %r, %r)" % (
  156. self.text,
  157. self.escapes_code.args,
  158. (self.lineno, self.pos),
  159. )
  160. class _TagMeta(type):
  161. """metaclass to allow Tag to produce a subclass according to
  162. its keyword"""
  163. _classmap = {}
  164. def __init__(cls, clsname, bases, dict_):
  165. if getattr(cls, "__keyword__", None) is not None:
  166. cls._classmap[cls.__keyword__] = cls
  167. super().__init__(clsname, bases, dict_)
  168. def __call__(cls, keyword, attributes, **kwargs):
  169. if ":" in keyword:
  170. ns, defname = keyword.split(":")
  171. return type.__call__(
  172. CallNamespaceTag, ns, defname, attributes, **kwargs
  173. )
  174. try:
  175. cls = _TagMeta._classmap[keyword]
  176. except KeyError:
  177. raise exceptions.CompileException(
  178. "No such tag: '%s'" % keyword,
  179. source=kwargs["source"],
  180. lineno=kwargs["lineno"],
  181. pos=kwargs["pos"],
  182. filename=kwargs["filename"],
  183. )
  184. return type.__call__(cls, keyword, attributes, **kwargs)
  185. class Tag(Node, metaclass=_TagMeta):
  186. """abstract base class for tags.
  187. e.g.::
  188. <%sometag/>
  189. <%someothertag>
  190. stuff
  191. </%someothertag>
  192. """
  193. __keyword__ = None
  194. def __init__(
  195. self,
  196. keyword,
  197. attributes,
  198. expressions,
  199. nonexpressions,
  200. required,
  201. **kwargs,
  202. ):
  203. r"""construct a new Tag instance.
  204. this constructor not called directly, and is only called
  205. by subclasses.
  206. :param keyword: the tag keyword
  207. :param attributes: raw dictionary of attribute key/value pairs
  208. :param expressions: a set of identifiers that are legal attributes,
  209. which can also contain embedded expressions
  210. :param nonexpressions: a set of identifiers that are legal
  211. attributes, which cannot contain embedded expressions
  212. :param \**kwargs:
  213. other arguments passed to the Node superclass (lineno, pos)
  214. """
  215. super().__init__(**kwargs)
  216. self.keyword = keyword
  217. self.attributes = attributes
  218. self._parse_attributes(expressions, nonexpressions)
  219. missing = [r for r in required if r not in self.parsed_attributes]
  220. if len(missing):
  221. raise exceptions.CompileException(
  222. (
  223. "Missing attribute(s): %s"
  224. % ",".join(repr(m) for m in missing)
  225. ),
  226. **self.exception_kwargs,
  227. )
  228. self.parent = None
  229. self.nodes = []
  230. def is_root(self):
  231. return self.parent is None
  232. def get_children(self):
  233. return self.nodes
  234. def _parse_attributes(self, expressions, nonexpressions):
  235. undeclared_identifiers = set()
  236. self.parsed_attributes = {}
  237. for key in self.attributes:
  238. if key in expressions:
  239. expr = []
  240. for x in re.compile(r"(\${(?:[^$]*?{.+|.+?)})", re.S).split(
  241. self.attributes[key]
  242. ):
  243. m = re.compile(r"^\${(.+?)}$", re.S).match(x)
  244. if m:
  245. code = ast.PythonCode(
  246. m.group(1).rstrip(), **self.exception_kwargs
  247. )
  248. # we aren't discarding "declared_identifiers" here,
  249. # which we do so that list comprehension-declared
  250. # variables aren't counted. As yet can't find a
  251. # condition that requires it here.
  252. undeclared_identifiers = undeclared_identifiers.union(
  253. code.undeclared_identifiers
  254. )
  255. expr.append("(%s)" % m.group(1))
  256. elif x:
  257. expr.append(repr(x))
  258. self.parsed_attributes[key] = " + ".join(expr) or repr("")
  259. elif key in nonexpressions:
  260. if re.search(r"\${.+?}", self.attributes[key]):
  261. raise exceptions.CompileException(
  262. "Attribute '%s' in tag '%s' does not allow embedded "
  263. "expressions" % (key, self.keyword),
  264. **self.exception_kwargs,
  265. )
  266. self.parsed_attributes[key] = repr(self.attributes[key])
  267. else:
  268. raise exceptions.CompileException(
  269. "Invalid attribute for tag '%s': '%s'"
  270. % (self.keyword, key),
  271. **self.exception_kwargs,
  272. )
  273. self.expression_undeclared_identifiers = undeclared_identifiers
  274. def declared_identifiers(self):
  275. return []
  276. def undeclared_identifiers(self):
  277. return self.expression_undeclared_identifiers
  278. def __repr__(self):
  279. return "%s(%r, %s, %r, %r)" % (
  280. self.__class__.__name__,
  281. self.keyword,
  282. util.sorted_dict_repr(self.attributes),
  283. (self.lineno, self.pos),
  284. self.nodes,
  285. )
  286. class IncludeTag(Tag):
  287. __keyword__ = "include"
  288. def __init__(self, keyword, attributes, **kwargs):
  289. super().__init__(
  290. keyword,
  291. attributes,
  292. ("file", "import", "args"),
  293. (),
  294. ("file",),
  295. **kwargs,
  296. )
  297. self.page_args = ast.PythonCode(
  298. "__DUMMY(%s)" % attributes.get("args", ""), **self.exception_kwargs
  299. )
  300. def declared_identifiers(self):
  301. return []
  302. def undeclared_identifiers(self):
  303. identifiers = self.page_args.undeclared_identifiers.difference(
  304. {"__DUMMY"}
  305. ).difference(self.page_args.declared_identifiers)
  306. return identifiers.union(super().undeclared_identifiers())
  307. class NamespaceTag(Tag):
  308. __keyword__ = "namespace"
  309. def __init__(self, keyword, attributes, **kwargs):
  310. super().__init__(
  311. keyword,
  312. attributes,
  313. ("file",),
  314. ("name", "inheritable", "import", "module"),
  315. (),
  316. **kwargs,
  317. )
  318. self.name = attributes.get("name", "__anon_%s" % hex(abs(id(self))))
  319. if "name" not in attributes and "import" not in attributes:
  320. raise exceptions.CompileException(
  321. "'name' and/or 'import' attributes are required "
  322. "for <%namespace>",
  323. **self.exception_kwargs,
  324. )
  325. if "file" in attributes and "module" in attributes:
  326. raise exceptions.CompileException(
  327. "<%namespace> may only have one of 'file' or 'module'",
  328. **self.exception_kwargs,
  329. )
  330. def declared_identifiers(self):
  331. return []
  332. class TextTag(Tag):
  333. __keyword__ = "text"
  334. def __init__(self, keyword, attributes, **kwargs):
  335. super().__init__(keyword, attributes, (), ("filter"), (), **kwargs)
  336. self.filter_args = ast.ArgumentList(
  337. attributes.get("filter", ""), **self.exception_kwargs
  338. )
  339. def undeclared_identifiers(self):
  340. return self.filter_args.undeclared_identifiers.difference(
  341. filters.DEFAULT_ESCAPES.keys()
  342. ).union(self.expression_undeclared_identifiers)
  343. class DefTag(Tag):
  344. __keyword__ = "def"
  345. def __init__(self, keyword, attributes, **kwargs):
  346. expressions = ["buffered", "cached"] + [
  347. c for c in attributes if c.startswith("cache_")
  348. ]
  349. super().__init__(
  350. keyword,
  351. attributes,
  352. expressions,
  353. ("name", "filter", "decorator"),
  354. ("name",),
  355. **kwargs,
  356. )
  357. name = attributes["name"]
  358. if re.match(r"^[\w_]+$", name):
  359. raise exceptions.CompileException(
  360. "Missing parenthesis in %def", **self.exception_kwargs
  361. )
  362. self.function_decl = ast.FunctionDecl(
  363. "def " + name + ":pass", **self.exception_kwargs
  364. )
  365. self.name = self.function_decl.funcname
  366. self.decorator = attributes.get("decorator", "")
  367. self.filter_args = ast.ArgumentList(
  368. attributes.get("filter", ""), **self.exception_kwargs
  369. )
  370. is_anonymous = False
  371. is_block = False
  372. @property
  373. def funcname(self):
  374. return self.function_decl.funcname
  375. def get_argument_expressions(self, **kw):
  376. return self.function_decl.get_argument_expressions(**kw)
  377. def declared_identifiers(self):
  378. return self.function_decl.allargnames
  379. def undeclared_identifiers(self):
  380. res = []
  381. for c in self.function_decl.defaults:
  382. res += list(
  383. ast.PythonCode(
  384. c, **self.exception_kwargs
  385. ).undeclared_identifiers
  386. )
  387. return (
  388. set(res)
  389. .union(
  390. self.filter_args.undeclared_identifiers.difference(
  391. filters.DEFAULT_ESCAPES.keys()
  392. )
  393. )
  394. .union(self.expression_undeclared_identifiers)
  395. .difference(self.function_decl.allargnames)
  396. )
  397. class BlockTag(Tag):
  398. __keyword__ = "block"
  399. def __init__(self, keyword, attributes, **kwargs):
  400. expressions = ["buffered", "cached", "args"] + [
  401. c for c in attributes if c.startswith("cache_")
  402. ]
  403. super().__init__(
  404. keyword,
  405. attributes,
  406. expressions,
  407. ("name", "filter", "decorator"),
  408. (),
  409. **kwargs,
  410. )
  411. name = attributes.get("name")
  412. if name and not re.match(r"^[\w_]+$", name):
  413. raise exceptions.CompileException(
  414. "%block may not specify an argument signature",
  415. **self.exception_kwargs,
  416. )
  417. if not name and attributes.get("args", None):
  418. raise exceptions.CompileException(
  419. "Only named %blocks may specify args", **self.exception_kwargs
  420. )
  421. self.body_decl = ast.FunctionArgs(
  422. attributes.get("args", ""), **self.exception_kwargs
  423. )
  424. self.name = name
  425. self.decorator = attributes.get("decorator", "")
  426. self.filter_args = ast.ArgumentList(
  427. attributes.get("filter", ""), **self.exception_kwargs
  428. )
  429. is_block = True
  430. @property
  431. def is_anonymous(self):
  432. return self.name is None
  433. @property
  434. def funcname(self):
  435. return self.name or "__M_anon_%d" % (self.lineno,)
  436. def get_argument_expressions(self, **kw):
  437. return self.body_decl.get_argument_expressions(**kw)
  438. def declared_identifiers(self):
  439. return self.body_decl.allargnames
  440. def undeclared_identifiers(self):
  441. return (
  442. self.filter_args.undeclared_identifiers.difference(
  443. filters.DEFAULT_ESCAPES.keys()
  444. )
  445. ).union(self.expression_undeclared_identifiers)
  446. class CallTag(Tag):
  447. __keyword__ = "call"
  448. def __init__(self, keyword, attributes, **kwargs):
  449. super().__init__(
  450. keyword, attributes, ("args"), ("expr",), ("expr",), **kwargs
  451. )
  452. self.expression = attributes["expr"]
  453. self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
  454. self.body_decl = ast.FunctionArgs(
  455. attributes.get("args", ""), **self.exception_kwargs
  456. )
  457. def declared_identifiers(self):
  458. return self.code.declared_identifiers.union(self.body_decl.allargnames)
  459. def undeclared_identifiers(self):
  460. return self.code.undeclared_identifiers.difference(
  461. self.code.declared_identifiers
  462. )
  463. class CallNamespaceTag(Tag):
  464. def __init__(self, namespace, defname, attributes, **kwargs):
  465. super().__init__(
  466. namespace + ":" + defname,
  467. attributes,
  468. tuple(attributes.keys()) + ("args",),
  469. (),
  470. (),
  471. **kwargs,
  472. )
  473. self.expression = "%s.%s(%s)" % (
  474. namespace,
  475. defname,
  476. ",".join(
  477. "%s=%s" % (k, v)
  478. for k, v in self.parsed_attributes.items()
  479. if k != "args"
  480. ),
  481. )
  482. self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
  483. self.body_decl = ast.FunctionArgs(
  484. attributes.get("args", ""), **self.exception_kwargs
  485. )
  486. def declared_identifiers(self):
  487. return self.code.declared_identifiers.union(self.body_decl.allargnames)
  488. def undeclared_identifiers(self):
  489. return self.code.undeclared_identifiers.difference(
  490. self.code.declared_identifiers
  491. )
  492. class InheritTag(Tag):
  493. __keyword__ = "inherit"
  494. def __init__(self, keyword, attributes, **kwargs):
  495. super().__init__(
  496. keyword, attributes, ("file",), (), ("file",), **kwargs
  497. )
  498. class PageTag(Tag):
  499. __keyword__ = "page"
  500. def __init__(self, keyword, attributes, **kwargs):
  501. expressions = [
  502. "cached",
  503. "args",
  504. "expression_filter",
  505. "enable_loop",
  506. ] + [c for c in attributes if c.startswith("cache_")]
  507. super().__init__(keyword, attributes, expressions, (), (), **kwargs)
  508. self.body_decl = ast.FunctionArgs(
  509. attributes.get("args", ""), **self.exception_kwargs
  510. )
  511. self.filter_args = ast.ArgumentList(
  512. attributes.get("expression_filter", ""), **self.exception_kwargs
  513. )
  514. def declared_identifiers(self):
  515. return self.body_decl.allargnames