source: python/trunk/Lib/test/test_gc.py

Last change on this file was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 22.2 KB
Line 
1import unittest
2from test.test_support import verbose, run_unittest
3import sys
4import time
5import gc
6import weakref
7
8try:
9 import threading
10except ImportError:
11 threading = None
12
13### Support code
14###############################################################################
15
16# Bug 1055820 has several tests of longstanding bugs involving weakrefs and
17# cyclic gc.
18
19# An instance of C1055820 has a self-loop, so becomes cyclic trash when
20# unreachable.
21class C1055820(object):
22 def __init__(self, i):
23 self.i = i
24 self.loop = self
25
26class GC_Detector(object):
27 # Create an instance I. Then gc hasn't happened again so long as
28 # I.gc_happened is false.
29
30 def __init__(self):
31 self.gc_happened = False
32
33 def it_happened(ignored):
34 self.gc_happened = True
35
36 # Create a piece of cyclic trash that triggers it_happened when
37 # gc collects it.
38 self.wr = weakref.ref(C1055820(666), it_happened)
39
40
41### Tests
42###############################################################################
43
44class GCTests(unittest.TestCase):
45 def test_list(self):
46 l = []
47 l.append(l)
48 gc.collect()
49 del l
50 self.assertEqual(gc.collect(), 1)
51
52 def test_dict(self):
53 d = {}
54 d[1] = d
55 gc.collect()
56 del d
57 self.assertEqual(gc.collect(), 1)
58
59 def test_tuple(self):
60 # since tuples are immutable we close the loop with a list
61 l = []
62 t = (l,)
63 l.append(t)
64 gc.collect()
65 del t
66 del l
67 self.assertEqual(gc.collect(), 2)
68
69 def test_class(self):
70 class A:
71 pass
72 A.a = A
73 gc.collect()
74 del A
75 self.assertNotEqual(gc.collect(), 0)
76
77 def test_newstyleclass(self):
78 class A(object):
79 pass
80 gc.collect()
81 del A
82 self.assertNotEqual(gc.collect(), 0)
83
84 def test_instance(self):
85 class A:
86 pass
87 a = A()
88 a.a = a
89 gc.collect()
90 del a
91 self.assertNotEqual(gc.collect(), 0)
92
93 def test_newinstance(self):
94 class A(object):
95 pass
96 a = A()
97 a.a = a
98 gc.collect()
99 del a
100 self.assertNotEqual(gc.collect(), 0)
101 class B(list):
102 pass
103 class C(B, A):
104 pass
105 a = C()
106 a.a = a
107 gc.collect()
108 del a
109 self.assertNotEqual(gc.collect(), 0)
110 del B, C
111 self.assertNotEqual(gc.collect(), 0)
112 A.a = A()
113 del A
114 self.assertNotEqual(gc.collect(), 0)
115 self.assertEqual(gc.collect(), 0)
116
117 def test_method(self):
118 # Tricky: self.__init__ is a bound method, it references the instance.
119 class A:
120 def __init__(self):
121 self.init = self.__init__
122 a = A()
123 gc.collect()
124 del a
125 self.assertNotEqual(gc.collect(), 0)
126
127 def test_finalizer(self):
128 # A() is uncollectable if it is part of a cycle, make sure it shows up
129 # in gc.garbage.
130 class A:
131 def __del__(self): pass
132 class B:
133 pass
134 a = A()
135 a.a = a
136 id_a = id(a)
137 b = B()
138 b.b = b
139 gc.collect()
140 del a
141 del b
142 self.assertNotEqual(gc.collect(), 0)
143 for obj in gc.garbage:
144 if id(obj) == id_a:
145 del obj.a
146 break
147 else:
148 self.fail("didn't find obj in garbage (finalizer)")
149 gc.garbage.remove(obj)
150
151 def test_finalizer_newclass(self):
152 # A() is uncollectable if it is part of a cycle, make sure it shows up
153 # in gc.garbage.
154 class A(object):
155 def __del__(self): pass
156 class B(object):
157 pass
158 a = A()
159 a.a = a
160 id_a = id(a)
161 b = B()
162 b.b = b
163 gc.collect()
164 del a
165 del b
166 self.assertNotEqual(gc.collect(), 0)
167 for obj in gc.garbage:
168 if id(obj) == id_a:
169 del obj.a
170 break
171 else:
172 self.fail("didn't find obj in garbage (finalizer)")
173 gc.garbage.remove(obj)
174
175 def test_function(self):
176 # Tricky: f -> d -> f, code should call d.clear() after the exec to
177 # break the cycle.
178 d = {}
179 exec("def f(): pass\n") in d
180 gc.collect()
181 del d
182 self.assertEqual(gc.collect(), 2)
183
184 def test_frame(self):
185 def f():
186 frame = sys._getframe()
187 gc.collect()
188 f()
189 self.assertEqual(gc.collect(), 1)
190
191 def test_saveall(self):
192 # Verify that cyclic garbage like lists show up in gc.garbage if the
193 # SAVEALL option is enabled.
194
195 # First make sure we don't save away other stuff that just happens to
196 # be waiting for collection.
197 gc.collect()
198 # if this fails, someone else created immortal trash
199 self.assertEqual(gc.garbage, [])
200
201 L = []
202 L.append(L)
203 id_L = id(L)
204
205 debug = gc.get_debug()
206 gc.set_debug(debug | gc.DEBUG_SAVEALL)
207 del L
208 gc.collect()
209 gc.set_debug(debug)
210
211 self.assertEqual(len(gc.garbage), 1)
212 obj = gc.garbage.pop()
213 self.assertEqual(id(obj), id_L)
214
215 def test_del(self):
216 # __del__ methods can trigger collection, make this to happen
217 thresholds = gc.get_threshold()
218 gc.enable()
219 gc.set_threshold(1)
220
221 class A:
222 def __del__(self):
223 dir(self)
224 a = A()
225 del a
226
227 gc.disable()
228 gc.set_threshold(*thresholds)
229
230 def test_del_newclass(self):
231 # __del__ methods can trigger collection, make this to happen
232 thresholds = gc.get_threshold()
233 gc.enable()
234 gc.set_threshold(1)
235
236 class A(object):
237 def __del__(self):
238 dir(self)
239 a = A()
240 del a
241
242 gc.disable()
243 gc.set_threshold(*thresholds)
244
245 # The following two tests are fragile:
246 # They precisely count the number of allocations,
247 # which is highly implementation-dependent.
248 # For example:
249 # - disposed tuples are not freed, but reused
250 # - the call to assertEqual somehow avoids building its args tuple
251 def test_get_count(self):
252 # Avoid future allocation of method object
253 assertEqual = self._baseAssertEqual
254 gc.collect()
255 assertEqual(gc.get_count(), (0, 0, 0))
256 a = dict()
257 # since gc.collect(), we created two objects:
258 # the dict, and the tuple returned by get_count()
259 assertEqual(gc.get_count(), (2, 0, 0))
260
261 def test_collect_generations(self):
262 # Avoid future allocation of method object
263 assertEqual = self.assertEqual
264 gc.collect()
265 a = dict()
266 gc.collect(0)
267 assertEqual(gc.get_count(), (0, 1, 0))
268 gc.collect(1)
269 assertEqual(gc.get_count(), (0, 0, 1))
270 gc.collect(2)
271 assertEqual(gc.get_count(), (0, 0, 0))
272
273 def test_trashcan(self):
274 class Ouch:
275 n = 0
276 def __del__(self):
277 Ouch.n = Ouch.n + 1
278 if Ouch.n % 17 == 0:
279 gc.collect()
280
281 # "trashcan" is a hack to prevent stack overflow when deallocating
282 # very deeply nested tuples etc. It works in part by abusing the
283 # type pointer and refcount fields, and that can yield horrible
284 # problems when gc tries to traverse the structures.
285 # If this test fails (as it does in 2.0, 2.1 and 2.2), it will
286 # most likely die via segfault.
287
288 # Note: In 2.3 the possibility for compiling without cyclic gc was
289 # removed, and that in turn allows the trashcan mechanism to work
290 # via much simpler means (e.g., it never abuses the type pointer or
291 # refcount fields anymore). Since it's much less likely to cause a
292 # problem now, the various constants in this expensive (we force a lot
293 # of full collections) test are cut back from the 2.2 version.
294 gc.enable()
295 N = 150
296 for count in range(2):
297 t = []
298 for i in range(N):
299 t = [t, Ouch()]
300 u = []
301 for i in range(N):
302 u = [u, Ouch()]
303 v = {}
304 for i in range(N):
305 v = {1: v, 2: Ouch()}
306 gc.disable()
307
308 @unittest.skipUnless(threading, "test meaningless on builds without threads")
309 def test_trashcan_threads(self):
310 # Issue #13992: trashcan mechanism should be thread-safe
311 NESTING = 60
312 N_THREADS = 2
313
314 def sleeper_gen():
315 """A generator that releases the GIL when closed or dealloc'ed."""
316 try:
317 yield
318 finally:
319 time.sleep(0.000001)
320
321 class C(list):
322 # Appending to a list is atomic, which avoids the use of a lock.
323 inits = []
324 dels = []
325 def __init__(self, alist):
326 self[:] = alist
327 C.inits.append(None)
328 def __del__(self):
329 # This __del__ is called by subtype_dealloc().
330 C.dels.append(None)
331 # `g` will release the GIL when garbage-collected. This
332 # helps assert subtype_dealloc's behaviour when threads
333 # switch in the middle of it.
334 g = sleeper_gen()
335 next(g)
336 # Now that __del__ is finished, subtype_dealloc will proceed
337 # to call list_dealloc, which also uses the trashcan mechanism.
338
339 def make_nested():
340 """Create a sufficiently nested container object so that the
341 trashcan mechanism is invoked when deallocating it."""
342 x = C([])
343 for i in range(NESTING):
344 x = [C([x])]
345 del x
346
347 def run_thread():
348 """Exercise make_nested() in a loop."""
349 while not exit:
350 make_nested()
351
352 old_checkinterval = sys.getcheckinterval()
353 sys.setcheckinterval(3)
354 try:
355 exit = False
356 threads = []
357 for i in range(N_THREADS):
358 t = threading.Thread(target=run_thread)
359 threads.append(t)
360 for t in threads:
361 t.start()
362 time.sleep(1.0)
363 exit = True
364 for t in threads:
365 t.join()
366 finally:
367 sys.setcheckinterval(old_checkinterval)
368 gc.collect()
369 self.assertEqual(len(C.inits), len(C.dels))
370
371 def test_boom(self):
372 class Boom:
373 def __getattr__(self, someattribute):
374 del self.attr
375 raise AttributeError
376
377 a = Boom()
378 b = Boom()
379 a.attr = b
380 b.attr = a
381
382 gc.collect()
383 garbagelen = len(gc.garbage)
384 del a, b
385 # a<->b are in a trash cycle now. Collection will invoke
386 # Boom.__getattr__ (to see whether a and b have __del__ methods), and
387 # __getattr__ deletes the internal "attr" attributes as a side effect.
388 # That causes the trash cycle to get reclaimed via refcounts falling to
389 # 0, thus mutating the trash graph as a side effect of merely asking
390 # whether __del__ exists. This used to (before 2.3b1) crash Python.
391 # Now __getattr__ isn't called.
392 self.assertEqual(gc.collect(), 4)
393 self.assertEqual(len(gc.garbage), garbagelen)
394
395 def test_boom2(self):
396 class Boom2:
397 def __init__(self):
398 self.x = 0
399
400 def __getattr__(self, someattribute):
401 self.x += 1
402 if self.x > 1:
403 del self.attr
404 raise AttributeError
405
406 a = Boom2()
407 b = Boom2()
408 a.attr = b
409 b.attr = a
410
411 gc.collect()
412 garbagelen = len(gc.garbage)
413 del a, b
414 # Much like test_boom(), except that __getattr__ doesn't break the
415 # cycle until the second time gc checks for __del__. As of 2.3b1,
416 # there isn't a second time, so this simply cleans up the trash cycle.
417 # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get
418 # reclaimed this way.
419 self.assertEqual(gc.collect(), 4)
420 self.assertEqual(len(gc.garbage), garbagelen)
421
422 def test_boom_new(self):
423 # boom__new and boom2_new are exactly like boom and boom2, except use
424 # new-style classes.
425
426 class Boom_New(object):
427 def __getattr__(self, someattribute):
428 del self.attr
429 raise AttributeError
430
431 a = Boom_New()
432 b = Boom_New()
433 a.attr = b
434 b.attr = a
435
436 gc.collect()
437 garbagelen = len(gc.garbage)
438 del a, b
439 self.assertEqual(gc.collect(), 4)
440 self.assertEqual(len(gc.garbage), garbagelen)
441
442 def test_boom2_new(self):
443 class Boom2_New(object):
444 def __init__(self):
445 self.x = 0
446
447 def __getattr__(self, someattribute):
448 self.x += 1
449 if self.x > 1:
450 del self.attr
451 raise AttributeError
452
453 a = Boom2_New()
454 b = Boom2_New()
455 a.attr = b
456 b.attr = a
457
458 gc.collect()
459 garbagelen = len(gc.garbage)
460 del a, b
461 self.assertEqual(gc.collect(), 4)
462 self.assertEqual(len(gc.garbage), garbagelen)
463
464 def test_get_referents(self):
465 alist = [1, 3, 5]
466 got = gc.get_referents(alist)
467 got.sort()
468 self.assertEqual(got, alist)
469
470 atuple = tuple(alist)
471 got = gc.get_referents(atuple)
472 got.sort()
473 self.assertEqual(got, alist)
474
475 adict = {1: 3, 5: 7}
476 expected = [1, 3, 5, 7]
477 got = gc.get_referents(adict)
478 got.sort()
479 self.assertEqual(got, expected)
480
481 got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
482 got.sort()
483 self.assertEqual(got, [0, 0] + range(5))
484
485 self.assertEqual(gc.get_referents(1, 'a', 4j), [])
486
487 def test_is_tracked(self):
488 # Atomic built-in types are not tracked, user-defined objects and
489 # mutable containers are.
490 # NOTE: types with special optimizations (e.g. tuple) have tests
491 # in their own test files instead.
492 self.assertFalse(gc.is_tracked(None))
493 self.assertFalse(gc.is_tracked(1))
494 self.assertFalse(gc.is_tracked(1.0))
495 self.assertFalse(gc.is_tracked(1.0 + 5.0j))
496 self.assertFalse(gc.is_tracked(True))
497 self.assertFalse(gc.is_tracked(False))
498 self.assertFalse(gc.is_tracked("a"))
499 self.assertFalse(gc.is_tracked(u"a"))
500 self.assertFalse(gc.is_tracked(bytearray("a")))
501 self.assertFalse(gc.is_tracked(type))
502 self.assertFalse(gc.is_tracked(int))
503 self.assertFalse(gc.is_tracked(object))
504 self.assertFalse(gc.is_tracked(object()))
505
506 class OldStyle:
507 pass
508 class NewStyle(object):
509 pass
510 self.assertTrue(gc.is_tracked(gc))
511 self.assertTrue(gc.is_tracked(OldStyle))
512 self.assertTrue(gc.is_tracked(OldStyle()))
513 self.assertTrue(gc.is_tracked(NewStyle))
514 self.assertTrue(gc.is_tracked(NewStyle()))
515 self.assertTrue(gc.is_tracked([]))
516 self.assertTrue(gc.is_tracked(set()))
517
518 def test_bug1055820b(self):
519 # Corresponds to temp2b.py in the bug report.
520
521 ouch = []
522 def callback(ignored):
523 ouch[:] = [wr() for wr in WRs]
524
525 Cs = [C1055820(i) for i in range(2)]
526 WRs = [weakref.ref(c, callback) for c in Cs]
527 c = None
528
529 gc.collect()
530 self.assertEqual(len(ouch), 0)
531 # Make the two instances trash, and collect again. The bug was that
532 # the callback materialized a strong reference to an instance, but gc
533 # cleared the instance's dict anyway.
534 Cs = None
535 gc.collect()
536 self.assertEqual(len(ouch), 2) # else the callbacks didn't run
537 for x in ouch:
538 # If the callback resurrected one of these guys, the instance
539 # would be damaged, with an empty __dict__.
540 self.assertEqual(x, None)
541
542class GCTogglingTests(unittest.TestCase):
543 def setUp(self):
544 gc.enable()
545
546 def tearDown(self):
547 gc.disable()
548
549 def test_bug1055820c(self):
550 # Corresponds to temp2c.py in the bug report. This is pretty
551 # elaborate.
552
553 c0 = C1055820(0)
554 # Move c0 into generation 2.
555 gc.collect()
556
557 c1 = C1055820(1)
558 c1.keep_c0_alive = c0
559 del c0.loop # now only c1 keeps c0 alive
560
561 c2 = C1055820(2)
562 c2wr = weakref.ref(c2) # no callback!
563
564 ouch = []
565 def callback(ignored):
566 ouch[:] = [c2wr()]
567
568 # The callback gets associated with a wr on an object in generation 2.
569 c0wr = weakref.ref(c0, callback)
570
571 c0 = c1 = c2 = None
572
573 # What we've set up: c0, c1, and c2 are all trash now. c0 is in
574 # generation 2. The only thing keeping it alive is that c1 points to
575 # it. c1 and c2 are in generation 0, and are in self-loops. There's a
576 # global weakref to c2 (c2wr), but that weakref has no callback.
577 # There's also a global weakref to c0 (c0wr), and that does have a
578 # callback, and that callback references c2 via c2wr().
579 #
580 # c0 has a wr with callback, which references c2wr
581 # ^
582 # |
583 # | Generation 2 above dots
584 #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
585 # | Generation 0 below dots
586 # |
587 # |
588 # ^->c1 ^->c2 has a wr but no callback
589 # | | | |
590 # <--v <--v
591 #
592 # So this is the nightmare: when generation 0 gets collected, we see
593 # that c2 has a callback-free weakref, and c1 doesn't even have a
594 # weakref. Collecting generation 0 doesn't see c0 at all, and c0 is
595 # the only object that has a weakref with a callback. gc clears c1
596 # and c2. Clearing c1 has the side effect of dropping the refcount on
597 # c0 to 0, so c0 goes away (despite that it's in an older generation)
598 # and c0's wr callback triggers. That in turn materializes a reference
599 # to c2 via c2wr(), but c2 gets cleared anyway by gc.
600
601 # We want to let gc happen "naturally", to preserve the distinction
602 # between generations.
603 junk = []
604 i = 0
605 detector = GC_Detector()
606 while not detector.gc_happened:
607 i += 1
608 if i > 10000:
609 self.fail("gc didn't happen after 10000 iterations")
610 self.assertEqual(len(ouch), 0)
611 junk.append([]) # this will eventually trigger gc
612
613 self.assertEqual(len(ouch), 1) # else the callback wasn't invoked
614 for x in ouch:
615 # If the callback resurrected c2, the instance would be damaged,
616 # with an empty __dict__.
617 self.assertEqual(x, None)
618
619 def test_bug1055820d(self):
620 # Corresponds to temp2d.py in the bug report. This is very much like
621 # test_bug1055820c, but uses a __del__ method instead of a weakref
622 # callback to sneak in a resurrection of cyclic trash.
623
624 ouch = []
625 class D(C1055820):
626 def __del__(self):
627 ouch[:] = [c2wr()]
628
629 d0 = D(0)
630 # Move all the above into generation 2.
631 gc.collect()
632
633 c1 = C1055820(1)
634 c1.keep_d0_alive = d0
635 del d0.loop # now only c1 keeps d0 alive
636
637 c2 = C1055820(2)
638 c2wr = weakref.ref(c2) # no callback!
639
640 d0 = c1 = c2 = None
641
642 # What we've set up: d0, c1, and c2 are all trash now. d0 is in
643 # generation 2. The only thing keeping it alive is that c1 points to
644 # it. c1 and c2 are in generation 0, and are in self-loops. There's
645 # a global weakref to c2 (c2wr), but that weakref has no callback.
646 # There are no other weakrefs.
647 #
648 # d0 has a __del__ method that references c2wr
649 # ^
650 # |
651 # | Generation 2 above dots
652 #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
653 # | Generation 0 below dots
654 # |
655 # |
656 # ^->c1 ^->c2 has a wr but no callback
657 # | | | |
658 # <--v <--v
659 #
660 # So this is the nightmare: when generation 0 gets collected, we see
661 # that c2 has a callback-free weakref, and c1 doesn't even have a
662 # weakref. Collecting generation 0 doesn't see d0 at all. gc clears
663 # c1 and c2. Clearing c1 has the side effect of dropping the refcount
664 # on d0 to 0, so d0 goes away (despite that it's in an older
665 # generation) and d0's __del__ triggers. That in turn materializes
666 # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc.
667
668 # We want to let gc happen "naturally", to preserve the distinction
669 # between generations.
670 detector = GC_Detector()
671 junk = []
672 i = 0
673 while not detector.gc_happened:
674 i += 1
675 if i > 10000:
676 self.fail("gc didn't happen after 10000 iterations")
677 self.assertEqual(len(ouch), 0)
678 junk.append([]) # this will eventually trigger gc
679
680 self.assertEqual(len(ouch), 1) # else __del__ wasn't invoked
681 for x in ouch:
682 # If __del__ resurrected c2, the instance would be damaged, with an
683 # empty __dict__.
684 self.assertEqual(x, None)
685
686def test_main():
687 enabled = gc.isenabled()
688 gc.disable()
689 assert not gc.isenabled()
690 debug = gc.get_debug()
691 gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak
692
693 try:
694 gc.collect() # Delete 2nd generation garbage
695 run_unittest(GCTests, GCTogglingTests)
696 finally:
697 gc.set_debug(debug)
698 # test gc.enable() even if GC is disabled by default
699 if verbose:
700 print "restoring automatic collection"
701 # make sure to always test gc.enable()
702 gc.enable()
703 assert gc.isenabled()
704 if not enabled:
705 gc.disable()
706
707if __name__ == "__main__":
708 test_main()
Note: See TracBrowser for help on using the repository browser.