__init__.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. # -*- coding: utf-8 -*-
  2. """
  3. Tests for greenlet.
  4. """
  5. import os
  6. import sys
  7. import sysconfig
  8. import unittest
  9. from gc import collect
  10. from gc import get_objects
  11. from threading import active_count as active_thread_count
  12. from time import sleep
  13. from time import time
  14. import psutil
  15. from greenlet import greenlet as RawGreenlet
  16. from greenlet import getcurrent
  17. from greenlet._greenlet import get_pending_cleanup_count
  18. from greenlet._greenlet import get_total_main_greenlets
  19. from . import leakcheck
  20. PY312 = sys.version_info[:2] >= (3, 12)
  21. PY313 = sys.version_info[:2] >= (3, 13)
  22. # XXX: First tested on 3.14a7. Revisit all uses of this on later versions to ensure they
  23. # are still valid.
  24. PY314 = sys.version_info[:2] >= (3, 14)
  25. WIN = sys.platform.startswith("win")
  26. RUNNING_ON_GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS')
  27. RUNNING_ON_TRAVIS = os.environ.get('TRAVIS') or RUNNING_ON_GITHUB_ACTIONS
  28. RUNNING_ON_APPVEYOR = os.environ.get('APPVEYOR')
  29. RUNNING_ON_CI = RUNNING_ON_TRAVIS or RUNNING_ON_APPVEYOR
  30. RUNNING_ON_MANYLINUX = os.environ.get('GREENLET_MANYLINUX')
  31. # Is the current interpreter free-threaded?) Note that this
  32. # isn't the same as whether the GIL is enabled, this is the build-time
  33. # value. Certain CPython details, like the garbage collector,
  34. # work very differently on potentially-free-threaded builds than
  35. # standard builds.
  36. RUNNING_ON_FREETHREAD_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
  37. class TestCaseMetaClass(type):
  38. # wrap each test method with
  39. # a) leak checks
  40. def __new__(cls, classname, bases, classDict):
  41. # pylint and pep8 fight over what this should be called (mcs or cls).
  42. # pylint gets it right, but we can't scope disable pep8, so we go with
  43. # its convention.
  44. # pylint: disable=bad-mcs-classmethod-argument
  45. check_totalrefcount = True
  46. # Python 3: must copy, we mutate the classDict. Interestingly enough,
  47. # it doesn't actually error out, but under 3.6 we wind up wrapping
  48. # and re-wrapping the same items over and over and over.
  49. for key, value in list(classDict.items()):
  50. if key.startswith('test') and callable(value):
  51. classDict.pop(key)
  52. if check_totalrefcount:
  53. value = leakcheck.wrap_refcount(value)
  54. classDict[key] = value
  55. return type.__new__(cls, classname, bases, classDict)
  56. class TestCase(unittest.TestCase, metaclass=TestCaseMetaClass):
  57. cleanup_attempt_sleep_duration = 0.001
  58. cleanup_max_sleep_seconds = 1
  59. def wait_for_pending_cleanups(self,
  60. initial_active_threads=None,
  61. initial_main_greenlets=None):
  62. initial_active_threads = initial_active_threads or self.threads_before_test
  63. initial_main_greenlets = initial_main_greenlets or self.main_greenlets_before_test
  64. sleep_time = self.cleanup_attempt_sleep_duration
  65. # NOTE: This is racy! A Python-level thread object may be dead
  66. # and gone, but the C thread may not yet have fired its
  67. # destructors and added to the queue. There's no particular
  68. # way to know that's about to happen. We try to watch the
  69. # Python threads to make sure they, at least, have gone away.
  70. # Counting the main greenlets, which we can easily do deterministically,
  71. # also helps.
  72. # Always sleep at least once to let other threads run
  73. sleep(sleep_time)
  74. quit_after = time() + self.cleanup_max_sleep_seconds
  75. # TODO: We could add an API that calls us back when a particular main greenlet is deleted?
  76. # It would have to drop the GIL
  77. while (
  78. get_pending_cleanup_count()
  79. or active_thread_count() > initial_active_threads
  80. or (not self.expect_greenlet_leak
  81. and get_total_main_greenlets() > initial_main_greenlets)):
  82. sleep(sleep_time)
  83. if time() > quit_after:
  84. print("Time limit exceeded.")
  85. print("Threads: Waiting for only", initial_active_threads,
  86. "-->", active_thread_count())
  87. print("MGlets : Waiting for only", initial_main_greenlets,
  88. "-->", get_total_main_greenlets())
  89. break
  90. collect()
  91. def count_objects(self, kind=list, exact_kind=True):
  92. # pylint:disable=unidiomatic-typecheck
  93. # Collect the garbage.
  94. for _ in range(3):
  95. collect()
  96. if exact_kind:
  97. return sum(
  98. 1
  99. for x in get_objects()
  100. if type(x) is kind
  101. )
  102. # instances
  103. return sum(
  104. 1
  105. for x in get_objects()
  106. if isinstance(x, kind)
  107. )
  108. greenlets_before_test = 0
  109. threads_before_test = 0
  110. main_greenlets_before_test = 0
  111. expect_greenlet_leak = False
  112. def count_greenlets(self):
  113. """
  114. Find all the greenlets and subclasses tracked by the GC.
  115. """
  116. return self.count_objects(RawGreenlet, False)
  117. def setUp(self):
  118. # Ensure the main greenlet exists, otherwise the first test
  119. # gets a false positive leak
  120. super().setUp()
  121. getcurrent()
  122. self.threads_before_test = active_thread_count()
  123. self.main_greenlets_before_test = get_total_main_greenlets()
  124. self.wait_for_pending_cleanups(self.threads_before_test, self.main_greenlets_before_test)
  125. self.greenlets_before_test = self.count_greenlets()
  126. def tearDown(self):
  127. if getattr(self, 'skipTearDown', False):
  128. return
  129. self.wait_for_pending_cleanups(self.threads_before_test, self.main_greenlets_before_test)
  130. super().tearDown()
  131. def get_expected_returncodes_for_aborted_process(self):
  132. import signal
  133. # The child should be aborted in an unusual way. On POSIX
  134. # platforms, this is done with abort() and signal.SIGABRT,
  135. # which is reflected in a negative return value; however, on
  136. # Windows, even though we observe the child print "Fatal
  137. # Python error: Aborted" and in older versions of the C
  138. # runtime "This application has requested the Runtime to
  139. # terminate it in an unusual way," it always has an exit code
  140. # of 3. This is interesting because 3 is the error code for
  141. # ERROR_PATH_NOT_FOUND; BUT: the C runtime abort() function
  142. # also uses this code.
  143. #
  144. # If we link to the static C library on Windows, the error
  145. # code changes to '0xc0000409' (hex(3221226505)), which
  146. # apparently is STATUS_STACK_BUFFER_OVERRUN; but "What this
  147. # means is that nowadays when you get a
  148. # STATUS_STACK_BUFFER_OVERRUN, it doesn’t actually mean that
  149. # there is a stack buffer overrun. It just means that the
  150. # application decided to terminate itself with great haste."
  151. #
  152. #
  153. # On windows, we've also seen '0xc0000005' (hex(3221225477)).
  154. # That's "Access Violation"
  155. #
  156. # See
  157. # https://devblogs.microsoft.com/oldnewthing/20110519-00/?p=10623
  158. # and
  159. # https://docs.microsoft.com/en-us/previous-versions/k089yyh0(v=vs.140)?redirectedfrom=MSDN
  160. # and
  161. # https://devblogs.microsoft.com/oldnewthing/20190108-00/?p=100655
  162. expected_exit = (
  163. -signal.SIGABRT,
  164. # But beginning on Python 3.11, the faulthandler
  165. # that prints the C backtraces sometimes segfaults after
  166. # reporting the exception but before printing the stack.
  167. # This has only been seen on linux/gcc.
  168. -signal.SIGSEGV,
  169. ) if not WIN else (
  170. 3,
  171. 0xc0000409,
  172. 0xc0000005,
  173. )
  174. return expected_exit
  175. def get_process_uss(self):
  176. """
  177. Return the current process's USS in bytes.
  178. uss is available on Linux, macOS, Windows. Also known as
  179. "Unique Set Size", this is the memory which is unique to a
  180. process and which would be freed if the process was terminated
  181. right now.
  182. If this is not supported by ``psutil``, this raises the
  183. :exc:`unittest.SkipTest` exception.
  184. """
  185. try:
  186. return psutil.Process().memory_full_info().uss
  187. except AttributeError as e:
  188. raise unittest.SkipTest("uss not supported") from e
  189. def run_script(self, script_name, show_output=True):
  190. import subprocess
  191. script = os.path.join(
  192. os.path.dirname(__file__),
  193. script_name,
  194. )
  195. try:
  196. return subprocess.check_output([sys.executable, script],
  197. encoding='utf-8',
  198. stderr=subprocess.STDOUT)
  199. except subprocess.CalledProcessError as ex:
  200. if show_output:
  201. print('-----')
  202. print('Failed to run script', script)
  203. print('~~~~~')
  204. print(ex.output)
  205. print('------')
  206. raise
  207. def assertScriptRaises(self, script_name, exitcodes=None):
  208. import subprocess
  209. with self.assertRaises(subprocess.CalledProcessError) as exc:
  210. output = self.run_script(script_name, show_output=False)
  211. __traceback_info__ = output
  212. # We're going to fail the assertion if we get here, at least
  213. # preserve the output in the traceback.
  214. if exitcodes is None:
  215. exitcodes = self.get_expected_returncodes_for_aborted_process()
  216. self.assertIn(exc.exception.returncode, exitcodes)
  217. return exc.exception