baked.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570
  1. # ext/baked.py
  2. # Copyright (C) 2005-2025 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. # mypy: ignore-errors
  8. """Baked query extension.
  9. Provides a creational pattern for the :class:`.query.Query` object which
  10. allows the fully constructed object, Core select statement, and string
  11. compiled result to be fully cached.
  12. """
  13. import collections.abc as collections_abc
  14. import logging
  15. from .. import exc as sa_exc
  16. from .. import util
  17. from ..orm import exc as orm_exc
  18. from ..orm.query import Query
  19. from ..orm.session import Session
  20. from ..sql import func
  21. from ..sql import literal_column
  22. from ..sql import util as sql_util
  23. log = logging.getLogger(__name__)
  24. class Bakery:
  25. """Callable which returns a :class:`.BakedQuery`.
  26. This object is returned by the class method
  27. :meth:`.BakedQuery.bakery`. It exists as an object
  28. so that the "cache" can be easily inspected.
  29. .. versionadded:: 1.2
  30. """
  31. __slots__ = "cls", "cache"
  32. def __init__(self, cls_, cache):
  33. self.cls = cls_
  34. self.cache = cache
  35. def __call__(self, initial_fn, *args):
  36. return self.cls(self.cache, initial_fn, args)
  37. class BakedQuery:
  38. """A builder object for :class:`.query.Query` objects."""
  39. __slots__ = "steps", "_bakery", "_cache_key", "_spoiled"
  40. def __init__(self, bakery, initial_fn, args=()):
  41. self._cache_key = ()
  42. self._update_cache_key(initial_fn, args)
  43. self.steps = [initial_fn]
  44. self._spoiled = False
  45. self._bakery = bakery
  46. @classmethod
  47. def bakery(cls, size=200, _size_alert=None):
  48. """Construct a new bakery.
  49. :return: an instance of :class:`.Bakery`
  50. """
  51. return Bakery(cls, util.LRUCache(size, size_alert=_size_alert))
  52. def _clone(self):
  53. b1 = BakedQuery.__new__(BakedQuery)
  54. b1._cache_key = self._cache_key
  55. b1.steps = list(self.steps)
  56. b1._bakery = self._bakery
  57. b1._spoiled = self._spoiled
  58. return b1
  59. def _update_cache_key(self, fn, args=()):
  60. self._cache_key += (fn.__code__,) + args
  61. def __iadd__(self, other):
  62. if isinstance(other, tuple):
  63. self.add_criteria(*other)
  64. else:
  65. self.add_criteria(other)
  66. return self
  67. def __add__(self, other):
  68. if isinstance(other, tuple):
  69. return self.with_criteria(*other)
  70. else:
  71. return self.with_criteria(other)
  72. def add_criteria(self, fn, *args):
  73. """Add a criteria function to this :class:`.BakedQuery`.
  74. This is equivalent to using the ``+=`` operator to
  75. modify a :class:`.BakedQuery` in-place.
  76. """
  77. self._update_cache_key(fn, args)
  78. self.steps.append(fn)
  79. return self
  80. def with_criteria(self, fn, *args):
  81. """Add a criteria function to a :class:`.BakedQuery` cloned from this
  82. one.
  83. This is equivalent to using the ``+`` operator to
  84. produce a new :class:`.BakedQuery` with modifications.
  85. """
  86. return self._clone().add_criteria(fn, *args)
  87. def for_session(self, session):
  88. """Return a :class:`_baked.Result` object for this
  89. :class:`.BakedQuery`.
  90. This is equivalent to calling the :class:`.BakedQuery` as a
  91. Python callable, e.g. ``result = my_baked_query(session)``.
  92. """
  93. return Result(self, session)
  94. def __call__(self, session):
  95. return self.for_session(session)
  96. def spoil(self, full=False):
  97. """Cancel any query caching that will occur on this BakedQuery object.
  98. The BakedQuery can continue to be used normally, however additional
  99. creational functions will not be cached; they will be called
  100. on every invocation.
  101. This is to support the case where a particular step in constructing
  102. a baked query disqualifies the query from being cacheable, such
  103. as a variant that relies upon some uncacheable value.
  104. :param full: if False, only functions added to this
  105. :class:`.BakedQuery` object subsequent to the spoil step will be
  106. non-cached; the state of the :class:`.BakedQuery` up until
  107. this point will be pulled from the cache. If True, then the
  108. entire :class:`_query.Query` object is built from scratch each
  109. time, with all creational functions being called on each
  110. invocation.
  111. """
  112. if not full and not self._spoiled:
  113. _spoil_point = self._clone()
  114. _spoil_point._cache_key += ("_query_only",)
  115. self.steps = [_spoil_point._retrieve_baked_query]
  116. self._spoiled = True
  117. return self
  118. def _effective_key(self, session):
  119. """Return the key that actually goes into the cache dictionary for
  120. this :class:`.BakedQuery`, taking into account the given
  121. :class:`.Session`.
  122. This basically means we also will include the session's query_class,
  123. as the actual :class:`_query.Query` object is part of what's cached
  124. and needs to match the type of :class:`_query.Query` that a later
  125. session will want to use.
  126. """
  127. return self._cache_key + (session._query_cls,)
  128. def _with_lazyload_options(self, options, effective_path, cache_path=None):
  129. """Cloning version of _add_lazyload_options."""
  130. q = self._clone()
  131. q._add_lazyload_options(options, effective_path, cache_path=cache_path)
  132. return q
  133. def _add_lazyload_options(self, options, effective_path, cache_path=None):
  134. """Used by per-state lazy loaders to add options to the
  135. "lazy load" query from a parent query.
  136. Creates a cache key based on given load path and query options;
  137. if a repeatable cache key cannot be generated, the query is
  138. "spoiled" so that it won't use caching.
  139. """
  140. key = ()
  141. if not cache_path:
  142. cache_path = effective_path
  143. for opt in options:
  144. if opt._is_legacy_option or opt._is_compile_state:
  145. ck = opt._generate_cache_key()
  146. if ck is None:
  147. self.spoil(full=True)
  148. else:
  149. assert not ck[1], (
  150. "loader options with variable bound parameters "
  151. "not supported with baked queries. Please "
  152. "use new-style select() statements for cached "
  153. "ORM queries."
  154. )
  155. key += ck[0]
  156. self.add_criteria(
  157. lambda q: q._with_current_path(effective_path).options(*options),
  158. cache_path.path,
  159. key,
  160. )
  161. def _retrieve_baked_query(self, session):
  162. query = self._bakery.get(self._effective_key(session), None)
  163. if query is None:
  164. query = self._as_query(session)
  165. self._bakery[self._effective_key(session)] = query.with_session(
  166. None
  167. )
  168. return query.with_session(session)
  169. def _bake(self, session):
  170. query = self._as_query(session)
  171. query.session = None
  172. # in 1.4, this is where before_compile() event is
  173. # invoked
  174. statement = query._statement_20()
  175. # if the query is not safe to cache, we still do everything as though
  176. # we did cache it, since the receiver of _bake() assumes subqueryload
  177. # context was set up, etc.
  178. #
  179. # note also we want to cache the statement itself because this
  180. # allows the statement itself to hold onto its cache key that is
  181. # used by the Connection, which in itself is more expensive to
  182. # generate than what BakedQuery was able to provide in 1.3 and prior
  183. if statement._compile_options._bake_ok:
  184. self._bakery[self._effective_key(session)] = (
  185. query,
  186. statement,
  187. )
  188. return query, statement
  189. def to_query(self, query_or_session):
  190. """Return the :class:`_query.Query` object for use as a subquery.
  191. This method should be used within the lambda callable being used
  192. to generate a step of an enclosing :class:`.BakedQuery`. The
  193. parameter should normally be the :class:`_query.Query` object that
  194. is passed to the lambda::
  195. sub_bq = self.bakery(lambda s: s.query(User.name))
  196. sub_bq += lambda q: q.filter(User.id == Address.user_id).correlate(Address)
  197. main_bq = self.bakery(lambda s: s.query(Address))
  198. main_bq += lambda q: q.filter(sub_bq.to_query(q).exists())
  199. In the case where the subquery is used in the first callable against
  200. a :class:`.Session`, the :class:`.Session` is also accepted::
  201. sub_bq = self.bakery(lambda s: s.query(User.name))
  202. sub_bq += lambda q: q.filter(User.id == Address.user_id).correlate(Address)
  203. main_bq = self.bakery(
  204. lambda s: s.query(Address.id, sub_bq.to_query(q).scalar_subquery())
  205. )
  206. :param query_or_session: a :class:`_query.Query` object or a class
  207. :class:`.Session` object, that is assumed to be within the context
  208. of an enclosing :class:`.BakedQuery` callable.
  209. .. versionadded:: 1.3
  210. """ # noqa: E501
  211. if isinstance(query_or_session, Session):
  212. session = query_or_session
  213. elif isinstance(query_or_session, Query):
  214. session = query_or_session.session
  215. if session is None:
  216. raise sa_exc.ArgumentError(
  217. "Given Query needs to be associated with a Session"
  218. )
  219. else:
  220. raise TypeError(
  221. "Query or Session object expected, got %r."
  222. % type(query_or_session)
  223. )
  224. return self._as_query(session)
  225. def _as_query(self, session):
  226. query = self.steps[0](session)
  227. for step in self.steps[1:]:
  228. query = step(query)
  229. return query
  230. class Result:
  231. """Invokes a :class:`.BakedQuery` against a :class:`.Session`.
  232. The :class:`_baked.Result` object is where the actual :class:`.query.Query`
  233. object gets created, or retrieved from the cache,
  234. against a target :class:`.Session`, and is then invoked for results.
  235. """
  236. __slots__ = "bq", "session", "_params", "_post_criteria"
  237. def __init__(self, bq, session):
  238. self.bq = bq
  239. self.session = session
  240. self._params = {}
  241. self._post_criteria = []
  242. def params(self, *args, **kw):
  243. """Specify parameters to be replaced into the string SQL statement."""
  244. if len(args) == 1:
  245. kw.update(args[0])
  246. elif len(args) > 0:
  247. raise sa_exc.ArgumentError(
  248. "params() takes zero or one positional argument, "
  249. "which is a dictionary."
  250. )
  251. self._params.update(kw)
  252. return self
  253. def _using_post_criteria(self, fns):
  254. if fns:
  255. self._post_criteria.extend(fns)
  256. return self
  257. def with_post_criteria(self, fn):
  258. """Add a criteria function that will be applied post-cache.
  259. This adds a function that will be run against the
  260. :class:`_query.Query` object after it is retrieved from the
  261. cache. This currently includes **only** the
  262. :meth:`_query.Query.params` and :meth:`_query.Query.execution_options`
  263. methods.
  264. .. warning:: :meth:`_baked.Result.with_post_criteria`
  265. functions are applied
  266. to the :class:`_query.Query`
  267. object **after** the query's SQL statement
  268. object has been retrieved from the cache. Only
  269. :meth:`_query.Query.params` and
  270. :meth:`_query.Query.execution_options`
  271. methods should be used.
  272. .. versionadded:: 1.2
  273. """
  274. return self._using_post_criteria([fn])
  275. def _as_query(self):
  276. q = self.bq._as_query(self.session).params(self._params)
  277. for fn in self._post_criteria:
  278. q = fn(q)
  279. return q
  280. def __str__(self):
  281. return str(self._as_query())
  282. def __iter__(self):
  283. return self._iter().__iter__()
  284. def _iter(self):
  285. bq = self.bq
  286. if not self.session.enable_baked_queries or bq._spoiled:
  287. return self._as_query()._iter()
  288. query, statement = bq._bakery.get(
  289. bq._effective_key(self.session), (None, None)
  290. )
  291. if query is None:
  292. query, statement = bq._bake(self.session)
  293. if self._params:
  294. q = query.params(self._params)
  295. else:
  296. q = query
  297. for fn in self._post_criteria:
  298. q = fn(q)
  299. params = q._params
  300. execution_options = dict(q._execution_options)
  301. execution_options.update(
  302. {
  303. "_sa_orm_load_options": q.load_options,
  304. "compiled_cache": bq._bakery,
  305. }
  306. )
  307. result = self.session.execute(
  308. statement, params, execution_options=execution_options
  309. )
  310. if result._attributes.get("is_single_entity", False):
  311. result = result.scalars()
  312. if result._attributes.get("filtered", False):
  313. result = result.unique()
  314. return result
  315. def count(self):
  316. """return the 'count'.
  317. Equivalent to :meth:`_query.Query.count`.
  318. Note this uses a subquery to ensure an accurate count regardless
  319. of the structure of the original statement.
  320. """
  321. col = func.count(literal_column("*"))
  322. bq = self.bq.with_criteria(lambda q: q._legacy_from_self(col))
  323. return bq.for_session(self.session).params(self._params).scalar()
  324. def scalar(self):
  325. """Return the first element of the first result or None
  326. if no rows present. If multiple rows are returned,
  327. raises MultipleResultsFound.
  328. Equivalent to :meth:`_query.Query.scalar`.
  329. """
  330. try:
  331. ret = self.one()
  332. if not isinstance(ret, collections_abc.Sequence):
  333. return ret
  334. return ret[0]
  335. except orm_exc.NoResultFound:
  336. return None
  337. def first(self):
  338. """Return the first row.
  339. Equivalent to :meth:`_query.Query.first`.
  340. """
  341. bq = self.bq.with_criteria(lambda q: q.slice(0, 1))
  342. return (
  343. bq.for_session(self.session)
  344. .params(self._params)
  345. ._using_post_criteria(self._post_criteria)
  346. ._iter()
  347. .first()
  348. )
  349. def one(self):
  350. """Return exactly one result or raise an exception.
  351. Equivalent to :meth:`_query.Query.one`.
  352. """
  353. return self._iter().one()
  354. def one_or_none(self):
  355. """Return one or zero results, or raise an exception for multiple
  356. rows.
  357. Equivalent to :meth:`_query.Query.one_or_none`.
  358. """
  359. return self._iter().one_or_none()
  360. def all(self):
  361. """Return all rows.
  362. Equivalent to :meth:`_query.Query.all`.
  363. """
  364. return self._iter().all()
  365. def get(self, ident):
  366. """Retrieve an object based on identity.
  367. Equivalent to :meth:`_query.Query.get`.
  368. """
  369. query = self.bq.steps[0](self.session)
  370. return query._get_impl(ident, self._load_on_pk_identity)
  371. def _load_on_pk_identity(self, session, query, primary_key_identity, **kw):
  372. """Load the given primary key identity from the database."""
  373. mapper = query._raw_columns[0]._annotations["parententity"]
  374. _get_clause, _get_params = mapper._get_clause
  375. def setup(query):
  376. _lcl_get_clause = _get_clause
  377. q = query._clone()
  378. q._get_condition()
  379. q._order_by = None
  380. # None present in ident - turn those comparisons
  381. # into "IS NULL"
  382. if None in primary_key_identity:
  383. nones = {
  384. _get_params[col].key
  385. for col, value in zip(
  386. mapper.primary_key, primary_key_identity
  387. )
  388. if value is None
  389. }
  390. _lcl_get_clause = sql_util.adapt_criterion_to_null(
  391. _lcl_get_clause, nones
  392. )
  393. # TODO: can mapper._get_clause be pre-adapted?
  394. q._where_criteria = (
  395. sql_util._deep_annotate(_lcl_get_clause, {"_orm_adapt": True}),
  396. )
  397. for fn in self._post_criteria:
  398. q = fn(q)
  399. return q
  400. # cache the query against a key that includes
  401. # which positions in the primary key are NULL
  402. # (remember, we can map to an OUTER JOIN)
  403. bq = self.bq
  404. # add the clause we got from mapper._get_clause to the cache
  405. # key so that if a race causes multiple calls to _get_clause,
  406. # we've cached on ours
  407. bq = bq._clone()
  408. bq._cache_key += (_get_clause,)
  409. bq = bq.with_criteria(
  410. setup, tuple(elem is None for elem in primary_key_identity)
  411. )
  412. params = {
  413. _get_params[primary_key].key: id_val
  414. for id_val, primary_key in zip(
  415. primary_key_identity, mapper.primary_key
  416. )
  417. }
  418. result = list(bq.for_session(self.session).params(**params))
  419. l = len(result)
  420. if l > 1:
  421. raise orm_exc.MultipleResultsFound()
  422. elif l:
  423. return result[0]
  424. else:
  425. return None
  426. bakery = BakedQuery.bakery