Changeset 391 for python/trunk/Lib/test/pickletester.py

Timestamp:

Mar 19, 2014, 11:31:01 PM (11 years ago)

Author:

dmik

Message:

python: Merge vendor 2.7.6 to trunk.

Location:

python/trunk

Files:

• . (modified) (1 prop)
• Lib/test/pickletester.py (modified) (27 diffs)

python/trunk/Lib/test/pickletester.py

@@ -3 +3 @@
 import cPickle
 import StringIO
+import cStringIO
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN, \
-                              run_with_locale
+from test.test_support import TestFailed, verbose, have_unicode, TESTFN
+try:
+    from test.test_support import _2G, _1M, precisionbigmemtest
+except ImportError:
+    # this import might fail when run on older Python versions by test_xpickle
+    _2G = _1M = 0
+    def precisionbigmemtest(*args, **kwargs):
+        return lambda self: None
 
 # Tests that try a number of pickle protocols should have a
@@ -14 +21 @@
 assert pickle.HIGHEST_PROTOCOL == cPickle.HIGHEST_PROTOCOL == 2
 protocols = range(pickle.HIGHEST_PROTOCOL + 1)
+
+# Copy of test.test_support.run_with_locale. This is needed to support Python
+# 2.4, which didn't include it. This is all to support test_xpickle, which
+# bounces pickled objects through older Python versions to test backwards
+# compatibility.
+def run_with_locale(catstr, *locales):
+    def decorator(func):
+        def inner(*args, **kwds):
+            try:
+                import locale
+                category = getattr(locale, catstr)
+                orig_locale = locale.setlocale(category)
+            except AttributeError:
+                # if the test author gives us an invalid category string
+                raise
+            except:
+                # cannot retrieve original locale, so do nothing
+                locale = orig_locale = None
+            else:
+                for loc in locales:
+                    try:
+                        locale.setlocale(category, loc)
+                        break
+                    except:
+                        pass
+
+            # now run the function, resetting the locale on exceptions
+            try:
+                return func(*args, **kwds)
+            finally:
+                if locale and orig_locale:
+                    locale.setlocale(category, orig_locale)
+        inner.func_name = func.func_name
+        inner.__doc__ = func.__doc__
+        return inner
+    return decorator
 
 
@@ -88 +131 @@
 class use_metaclass(object):
     __metaclass__ = metaclass
+
+class pickling_metaclass(type):
+    def __eq__(self, other):
+        return (type(self) == type(other) and
+                self.reduce_args == other.reduce_args)
+
+    def __reduce__(self):
+        return (create_dynamic_class, self.reduce_args)
+
+    __hash__ = None
+
+def create_dynamic_class(name, bases):
+    result = pickling_metaclass(name, bases, dict())
+    result.reduce_args = (name, bases)
+    return result
 
 # DATA0 .. DATA2 are the pickles we expect under the various protocols, for
@@ -411 +469 @@
     # of 1.
     def dont_test_disassembly(self):
-        from cStringIO import StringIO
         from pickletools import dis
 
         for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
             s = self.dumps(self._testdata, proto)
-            filelike = StringIO()
+            filelike = cStringIO.StringIO()
             dis(s, out=filelike)
             got = filelike.getvalue()
@@ -428 +485 @@
             x = self.loads(s)
             self.assertEqual(len(x), 1)
-            self.assert_(x is x[0])
+            self.assertTrue(x is x[0])
 
     def test_recursive_tuple(self):
@@ -438 +495 @@
             self.assertEqual(len(x), 1)
             self.assertEqual(len(x[0]), 1)
-            self.assert_(x is x[0][0])
+            self.assertTrue(x is x[0][0])
 
     def test_recursive_dict(self):
@@ -447 +504 @@
             x = self.loads(s)
             self.assertEqual(x.keys(), [1])
-            self.assert_(x[1] is x)
+            self.assertTrue(x[1] is x)
 
     def test_recursive_inst(self):
@@ -453 +510 @@
         i.attr = i
         for proto in protocols:
-            s = self.dumps(i, 2)
+            s = self.dumps(i, proto)
             x = self.loads(s)
             self.assertEqual(dir(x), dir(i))
-            self.assert_(x.attr is x)
+            self.assertIs(x.attr, x)
 
     def test_recursive_multi(self):
@@ -470 +527 @@
             self.assertEqual(dir(x[0]), dir(i))
             self.assertEqual(x[0].attr.keys(), [1])
-            self.assert_(x[0].attr[1] is x)
+            self.assertTrue(x[0].attr[1] is x)
 
     def test_garyp(self):
@@ -482 +539 @@
                     "'abc'   ?", # junk after close quote
                     "'\\'", # trailing backslash
+                    "'",    # issue #17710
+                    "' ",   # issue #17710
                     # some tests of the quoting rules
                     #"'abc\"\''",
@@ -575 +634 @@
             self.assertEqual(a.__class__, b.__class__)
 
+    def test_dynamic_class(self):
+        a = create_dynamic_class("my_dynamic_class", (object,))
+        copy_reg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
+        for proto in protocols:
+            s = self.dumps(a, proto)
+            b = self.loads(s)
+            self.assertEqual(a, b)
+
     def test_structseq(self):
         import time
@@ -611 +678 @@
             self.loads(badpickle)
         except ValueError, detail:
-            self.failUnless(str(detail).startswith(
+            self.assertTrue(str(detail).startswith(
                                             "unsupported pickle protocol"))
         else:
@@ -683 +750 @@
                 s = self.dumps(x, proto)
                 y = self.loads(s)
-                self.assert_(x is y, (proto, x, s, y))
+                self.assertTrue(x is y, (proto, x, s, y))
                 expected = expected_opcode[proto, x]
                 self.assertEqual(opcode_in_pickle(expected, s), True)
@@ -733 +800 @@
             # Dump using protocol 1 for comparison.
             s1 = self.dumps(x, 1)
-            self.assert_(__name__ in s1)
-            self.assert_("MyList" in s1)
+            self.assertIn(__name__, s1)
+            self.assertIn("MyList", s1)
             self.assertEqual(opcode_in_pickle(opcode, s1), False)
 
@@ -743 +810 @@
             # Dump using protocol 2 for test.
             s2 = self.dumps(x, 2)
-            self.assert_(__name__ not in s2)
-            self.assert_("MyList" not in s2)
+            self.assertNotIn(__name__, s2)
+            self.assertNotIn("MyList", s2)
             self.assertEqual(opcode_in_pickle(opcode, s2), True)
 
@@ -788 +855 @@
                 self.assertEqual(num_appends, 0)
             else:
-                self.failUnless(num_appends >= 2)
+                self.assertTrue(num_appends >= 2)
 
     def test_dict_chunking(self):
@@ -810 +877 @@
                 self.assertEqual(num_setitems, 0)
             else:
-                self.failUnless(num_setitems >= 2)
+                self.assertTrue(num_setitems >= 2)
 
     def test_simple_newobj(self):
@@ -834 +901 @@
 
     def test_reduce_overrides_default_reduce_ex(self):
-        for proto in 0, 1, 2:
+        for proto in protocols:
             x = REX_one()
             self.assertEqual(x._reduce_called, 0)
@@ -843 +910 @@
 
     def test_reduce_ex_called(self):
-        for proto in 0, 1, 2:
+        for proto in protocols:
             x = REX_two()
             self.assertEqual(x._proto, None)
@@ -852 +919 @@
 
     def test_reduce_ex_overrides_reduce(self):
-        for proto in 0, 1, 2:
+        for proto in protocols:
             x = REX_three()
             self.assertEqual(x._proto, None)
@@ -861 +928 @@
 
     def test_reduce_ex_calls_base(self):
-        for proto in 0, 1, 2:
+        for proto in protocols:
             x = REX_four()
             self.assertEqual(x._proto, None)
@@ -870 +937 @@
 
     def test_reduce_calls_base(self):
-        for proto in 0, 1, 2:
+        for proto in protocols:
             x = REX_five()
             self.assertEqual(x._reduce_called, 0)
@@ -891 +958 @@
 
         # Protocol 0 is less strict and also accept iterables.
-        for proto in 0, 1, 2:
+        for proto in protocols:
             try:
                 self.dumps(C(), proto)
@@ -901 +968 @@
                 pass
 
+    def test_many_puts_and_gets(self):
+        # Test that internal data structures correctly deal with lots of
+        # puts/gets.
+        keys = ("aaa" + str(i) for i in xrange(100))
+        large_dict = dict((k, [4, 5, 6]) for k in keys)
+        obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
+
+        for proto in protocols:
+            dumped = self.dumps(obj, proto)
+            loaded = self.loads(dumped)
+            self.assertEqual(loaded, obj,
+                             "Failed protocol %d: %r != %r"
+                             % (proto, obj, loaded))
+
+    def test_attribute_name_interning(self):
+        # Test that attribute names of pickled objects are interned when
+        # unpickling.
+        for proto in protocols:
+            x = C()
+            x.foo = 42
+            x.bar = "hello"
+            s = self.dumps(x, proto)
+            y = self.loads(s)
+            x_keys = sorted(x.__dict__)
+            y_keys = sorted(y.__dict__)
+            for x_key, y_key in zip(x_keys, y_keys):
+                self.assertIs(x_key, y_key)
+
+
 # Test classes for reduce_ex
 
@@ -1002 +1098 @@
             os.remove(TESTFN)
 
+    def test_load_from_and_dump_to_file(self):
+        stream = cStringIO.StringIO()
+        data = [123, {}, 124]
+        self.module.dump(data, stream)
+        stream.seek(0)
+        unpickled = self.module.load(stream)
+        self.assertEqual(unpickled, data)
+
     def test_highest_protocol(self):
         # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
@@ -1007 +1111 @@
 
     def test_callapi(self):
-        from cStringIO import StringIO
-        f = StringIO()
+        f = cStringIO.StringIO()
         # With and without keyword arguments
         self.module.dump(123, f, -1)
@@ -1074 +1177 @@
         self.assertEqual(self.id_count, 5)
         self.assertEqual(self.load_count, 5)
+
+class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
+
+    pickler_class = None
+    unpickler_class = None
+
+    def setUp(self):
+        assert self.pickler_class
+        assert self.unpickler_class
+
+    def test_clear_pickler_memo(self):
+        # To test whether clear_memo() has any effect, we pickle an object,
+        # then pickle it again without clearing the memo; the two serialized
+        # forms should be different. If we clear_memo() and then pickle the
+        # object again, the third serialized form should be identical to the
+        # first one we obtained.
+        data = ["abcdefg", "abcdefg", 44]
+        f = cStringIO.StringIO()
+        pickler = self.pickler_class(f)
+
+        pickler.dump(data)
+        first_pickled = f.getvalue()
+
+        # Reset StringIO object.
+        f.seek(0)
+        f.truncate()
+
+        pickler.dump(data)
+        second_pickled = f.getvalue()
+
+        # Reset the Pickler and StringIO objects.
+        pickler.clear_memo()
+        f.seek(0)
+        f.truncate()
+
+        pickler.dump(data)
+        third_pickled = f.getvalue()
+
+        self.assertNotEqual(first_pickled, second_pickled)
+        self.assertEqual(first_pickled, third_pickled)
+
+    def test_priming_pickler_memo(self):
+        # Verify that we can set the Pickler's memo attribute.
+        data = ["abcdefg", "abcdefg", 44]
+        f = cStringIO.StringIO()
+        pickler = self.pickler_class(f)
+
+        pickler.dump(data)
+        first_pickled = f.getvalue()
+
+        f = cStringIO.StringIO()
+        primed = self.pickler_class(f)
+        primed.memo = pickler.memo
+
+        primed.dump(data)
+        primed_pickled = f.getvalue()
+
+        self.assertNotEqual(first_pickled, primed_pickled)
+
+    def test_priming_unpickler_memo(self):
+        # Verify that we can set the Unpickler's memo attribute.
+        data = ["abcdefg", "abcdefg", 44]
+        f = cStringIO.StringIO()
+        pickler = self.pickler_class(f)
+
+        pickler.dump(data)
+        first_pickled = f.getvalue()
+
+        f = cStringIO.StringIO()
+        primed = self.pickler_class(f)
+        primed.memo = pickler.memo
+
+        primed.dump(data)
+        primed_pickled = f.getvalue()
+
+        unpickler = self.unpickler_class(cStringIO.StringIO(first_pickled))
+        unpickled_data1 = unpickler.load()
+
+        self.assertEqual(unpickled_data1, data)
+
+        primed = self.unpickler_class(cStringIO.StringIO(primed_pickled))
+        primed.memo = unpickler.memo
+        unpickled_data2 = primed.load()
+
+        primed.memo.clear()
+
+        self.assertEqual(unpickled_data2, data)
+        self.assertTrue(unpickled_data2 is unpickled_data1)
+
+    def test_reusing_unpickler_objects(self):
+        data1 = ["abcdefg", "abcdefg", 44]
+        f = cStringIO.StringIO()
+        pickler = self.pickler_class(f)
+        pickler.dump(data1)
+        pickled1 = f.getvalue()
+
+        data2 = ["abcdefg", 44, 44]
+        f = cStringIO.StringIO()
+        pickler = self.pickler_class(f)
+        pickler.dump(data2)
+        pickled2 = f.getvalue()
+
+        f = cStringIO.StringIO()
+        f.write(pickled1)
+        f.seek(0)
+        unpickler = self.unpickler_class(f)
+        self.assertEqual(unpickler.load(), data1)
+
+        f.seek(0)
+        f.truncate()
+        f.write(pickled2)
+        f.seek(0)
+        self.assertEqual(unpickler.load(), data2)
+
+class BigmemPickleTests(unittest.TestCase):
+
+    # Memory requirements: 1 byte per character for input strings, 1 byte
+    # for pickled data, 1 byte for unpickled strings, 1 byte for internal
+    # buffer and 1 byte of free space for resizing of internal buffer.
+
+    @precisionbigmemtest(size=_2G + 100*_1M, memuse=5)
+    def test_huge_strlist(self, size):
+        chunksize = 2**20
+        data = []
+        while size > chunksize:
+            data.append('x' * chunksize)
+            size -= chunksize
+            chunksize += 1
+        data.append('y' * size)
+
+        try:
+            for proto in protocols:
+                try:
+                    pickled = self.dumps(data, proto)
+                    res = self.loads(pickled)
+                    self.assertEqual(res, data)
+                finally:
+                    res = None
+                    pickled = None
+        finally:
+            data = None