source: python/trunk/Lib/test/test_codeccallbacks.py

Last change on this file was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 31.4 KB
Line 
1import test.test_support, unittest
2import sys, codecs, htmlentitydefs, unicodedata
3
4class PosReturn:
5 # this can be used for configurable callbacks
6
7 def __init__(self):
8 self.pos = 0
9
10 def handle(self, exc):
11 oldpos = self.pos
12 realpos = oldpos
13 if realpos<0:
14 realpos = len(exc.object) + realpos
15 # if we don't advance this time, terminate on the next call
16 # otherwise we'd get an endless loop
17 if realpos <= exc.start:
18 self.pos = len(exc.object)
19 return (u"<?>", oldpos)
20
21# A UnicodeEncodeError object with a bad start attribute
22class BadStartUnicodeEncodeError(UnicodeEncodeError):
23 def __init__(self):
24 UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
25 self.start = []
26
27# A UnicodeEncodeError object with a bad object attribute
28class BadObjectUnicodeEncodeError(UnicodeEncodeError):
29 def __init__(self):
30 UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
31 self.object = []
32
33# A UnicodeDecodeError object without an end attribute
34class NoEndUnicodeDecodeError(UnicodeDecodeError):
35 def __init__(self):
36 UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
37 del self.end
38
39# A UnicodeDecodeError object with a bad object attribute
40class BadObjectUnicodeDecodeError(UnicodeDecodeError):
41 def __init__(self):
42 UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
43 self.object = []
44
45# A UnicodeTranslateError object without a start attribute
46class NoStartUnicodeTranslateError(UnicodeTranslateError):
47 def __init__(self):
48 UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
49 del self.start
50
51# A UnicodeTranslateError object without an end attribute
52class NoEndUnicodeTranslateError(UnicodeTranslateError):
53 def __init__(self):
54 UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
55 del self.end
56
57# A UnicodeTranslateError object without an object attribute
58class NoObjectUnicodeTranslateError(UnicodeTranslateError):
59 def __init__(self):
60 UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
61 del self.object
62
63class CodecCallbackTest(unittest.TestCase):
64
65 def test_xmlcharrefreplace(self):
66 # replace unencodable characters which numeric character entities.
67 # For ascii, latin-1 and charmaps this is completely implemented
68 # in C and should be reasonably fast.
69 s = u"\u30b9\u30d1\u30e2 \xe4nd egg\u0161"
70 self.assertEqual(
71 s.encode("ascii", "xmlcharrefreplace"),
72 "&#12473;&#12497;&#12514; &#228;nd egg&#353;"
73 )
74 self.assertEqual(
75 s.encode("latin-1", "xmlcharrefreplace"),
76 "&#12473;&#12497;&#12514; \xe4nd egg&#353;"
77 )
78 self.assertEqual(
79 s.encode("iso-8859-15", "xmlcharrefreplace"),
80 "&#12473;&#12497;&#12514; \xe4nd egg\xa8"
81 )
82
83 def test_xmlcharrefreplace_with_surrogates(self):
84 tests = [(u'\U0001f49d', '&#128157;'),
85 (u'\ud83d', '&#55357;'),
86 (u'\udc9d', '&#56477;'),
87 ]
88 if u'\ud83d\udc9d' != u'\U0001f49d':
89 tests += [(u'\ud83d\udc9d', '&#55357;&#56477;')]
90 for encoding in ['ascii', 'latin1', 'iso-8859-15']:
91 for s, exp in tests:
92 self.assertEqual(s.encode(encoding, 'xmlcharrefreplace'),
93 exp, msg='%r.encode(%r)' % (s, encoding))
94 self.assertEqual((s+'X').encode(encoding, 'xmlcharrefreplace'),
95 exp+'X',
96 msg='%r.encode(%r)' % (s + 'X', encoding))
97
98 def test_xmlcharnamereplace(self):
99 # This time use a named character entity for unencodable
100 # characters, if one is available.
101
102 def xmlcharnamereplace(exc):
103 if not isinstance(exc, UnicodeEncodeError):
104 raise TypeError("don't know how to handle %r" % exc)
105 l = []
106 for c in exc.object[exc.start:exc.end]:
107 try:
108 l.append(u"&%s;" % htmlentitydefs.codepoint2name[ord(c)])
109 except KeyError:
110 l.append(u"&#%d;" % ord(c))
111 return (u"".join(l), exc.end)
112
113 codecs.register_error(
114 "test.xmlcharnamereplace", xmlcharnamereplace)
115
116 sin = u"\xab\u211c\xbb = \u2329\u1234\u20ac\u232a"
117 sout = "&laquo;&real;&raquo; = &lang;&#4660;&euro;&rang;"
118 self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout)
119 sout = "\xab&real;\xbb = &lang;&#4660;&euro;&rang;"
120 self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout)
121 sout = "\xab&real;\xbb = &lang;&#4660;\xa4&rang;"
122 self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout)
123
124 def test_uninamereplace(self):
125 # We're using the names from the unicode database this time,
126 # and we're doing "syntax highlighting" here, i.e. we include
127 # the replaced text in ANSI escape sequences. For this it is
128 # useful that the error handler is not called for every single
129 # unencodable character, but for a complete sequence of
130 # unencodable characters, otherwise we would output many
131 # unnecessary escape sequences.
132
133 def uninamereplace(exc):
134 if not isinstance(exc, UnicodeEncodeError):
135 raise TypeError("don't know how to handle %r" % exc)
136 l = []
137 for c in exc.object[exc.start:exc.end]:
138 l.append(unicodedata.name(c, u"0x%x" % ord(c)))
139 return (u"\033[1m%s\033[0m" % u", ".join(l), exc.end)
140
141 codecs.register_error(
142 "test.uninamereplace", uninamereplace)
143
144 sin = u"\xac\u1234\u20ac\u8000"
145 sout = "\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
146 self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout)
147
148 sout = "\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
149 self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout)
150
151 sout = "\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m"
152 self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout)
153
154 def test_backslashescape(self):
155 # Does the same as the "unicode-escape" encoding, but with different
156 # base encodings.
157 sin = u"a\xac\u1234\u20ac\u8000"
158 if sys.maxunicode > 0xffff:
159 sin += unichr(sys.maxunicode)
160 sout = "a\\xac\\u1234\\u20ac\\u8000"
161 if sys.maxunicode > 0xffff:
162 sout += "\\U%08x" % sys.maxunicode
163 self.assertEqual(sin.encode("ascii", "backslashreplace"), sout)
164
165 sout = "a\xac\\u1234\\u20ac\\u8000"
166 if sys.maxunicode > 0xffff:
167 sout += "\\U%08x" % sys.maxunicode
168 self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout)
169
170 sout = "a\xac\\u1234\xa4\\u8000"
171 if sys.maxunicode > 0xffff:
172 sout += "\\U%08x" % sys.maxunicode
173 self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout)
174
175 def test_decoding_callbacks(self):
176 # This is a test for a decoding callback handler
177 # that allows the decoding of the invalid sequence
178 # "\xc0\x80" and returns "\x00" instead of raising an error.
179 # All other illegal sequences will be handled strictly.
180 def relaxedutf8(exc):
181 if not isinstance(exc, UnicodeDecodeError):
182 raise TypeError("don't know how to handle %r" % exc)
183 if exc.object[exc.start:exc.start+2] == "\xc0\x80":
184 return (u"\x00", exc.start+2) # retry after two bytes
185 else:
186 raise exc
187
188 codecs.register_error("test.relaxedutf8", relaxedutf8)
189
190 # all the "\xc0\x80" will be decoded to "\x00"
191 sin = "a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80"
192 sout = u"a\x00b\x00c\xfc\x00\x00"
193 self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout)
194
195 # "\xc0\x81" is not valid and a UnicodeDecodeError will be raised
196 sin = "\xc0\x80\xc0\x81"
197 self.assertRaises(UnicodeDecodeError, sin.decode,
198 "utf-8", "test.relaxedutf8")
199
200 def test_charmapencode(self):
201 # For charmap encodings the replacement string will be
202 # mapped through the encoding again. This means, that
203 # to be able to use e.g. the "replace" handler, the
204 # charmap has to have a mapping for "?".
205 charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
206 sin = u"abc"
207 sout = "AABBCC"
208 self.assertEqual(codecs.charmap_encode(sin, "strict", charmap)[0], sout)
209
210 sin = u"abcA"
211 self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap)
212
213 charmap[ord("?")] = "XYZ"
214 sin = u"abcDEF"
215 sout = "AABBCCXYZXYZXYZ"
216 self.assertEqual(codecs.charmap_encode(sin, "replace", charmap)[0], sout)
217
218 charmap[ord("?")] = u"XYZ"
219 self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
220
221 charmap[ord("?")] = u"XYZ"
222 self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
223
224 def test_decodeunicodeinternal(self):
225 self.assertRaises(
226 UnicodeDecodeError,
227 "\x00\x00\x00\x00\x00".decode,
228 "unicode-internal",
229 )
230 if sys.maxunicode > 0xffff:
231 def handler_unicodeinternal(exc):
232 if not isinstance(exc, UnicodeDecodeError):
233 raise TypeError("don't know how to handle %r" % exc)
234 return (u"\x01", 1)
235
236 self.assertEqual(
237 "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
238 u"\u0000"
239 )
240
241 self.assertEqual(
242 "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"),
243 u"\u0000\ufffd"
244 )
245
246 codecs.register_error("test.hui", handler_unicodeinternal)
247
248 self.assertEqual(
249 "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"),
250 u"\u0000\u0001\u0000"
251 )
252
253 def test_callbacks(self):
254 def handler1(exc):
255 if not isinstance(exc, UnicodeEncodeError) \
256 and not isinstance(exc, UnicodeDecodeError):
257 raise TypeError("don't know how to handle %r" % exc)
258 l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
259 return (u"[%s]" % u"".join(l), exc.end)
260
261 codecs.register_error("test.handler1", handler1)
262
263 def handler2(exc):
264 if not isinstance(exc, UnicodeDecodeError):
265 raise TypeError("don't know how to handle %r" % exc)
266 l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
267 return (u"[%s]" % u"".join(l), exc.end+1) # skip one character
268
269 codecs.register_error("test.handler2", handler2)
270
271 s = "\x00\x81\x7f\x80\xff"
272
273 self.assertEqual(
274 s.decode("ascii", "test.handler1"),
275 u"\x00[<129>]\x7f[<128>][<255>]"
276 )
277 self.assertEqual(
278 s.decode("ascii", "test.handler2"),
279 u"\x00[<129>][<128>]"
280 )
281
282 self.assertEqual(
283 "\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
284 u"\u3042[<92><117><51>]xxx"
285 )
286
287 self.assertEqual(
288 "\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
289 u"\u3042[<92><117><51>]xx"
290 )
291
292 self.assertEqual(
293 codecs.charmap_decode("abc", "test.handler1", {ord("a"): u"z"})[0],
294 u"z[<98>][<99>]"
295 )
296
297 self.assertEqual(
298 u"g\xfc\xdfrk".encode("ascii", "test.handler1"),
299 u"g[<252><223>]rk"
300 )
301
302 self.assertEqual(
303 u"g\xfc\xdf".encode("ascii", "test.handler1"),
304 u"g[<252><223>]"
305 )
306
307 def test_longstrings(self):
308 # test long strings to check for memory overflow problems
309 errors = [ "strict", "ignore", "replace", "xmlcharrefreplace",
310 "backslashreplace"]
311 # register the handlers under different names,
312 # to prevent the codec from recognizing the name
313 for err in errors:
314 codecs.register_error("test." + err, codecs.lookup_error(err))
315 l = 1000
316 errors += [ "test." + err for err in errors ]
317 for uni in [ s*l for s in (u"x", u"\u3042", u"a\xe4") ]:
318 for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15",
319 "utf-8", "utf-7", "utf-16", "utf-32"):
320 for err in errors:
321 try:
322 uni.encode(enc, err)
323 except UnicodeError:
324 pass
325
326 def check_exceptionobjectargs(self, exctype, args, msg):
327 # Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion
328 # check with one missing argument
329 self.assertRaises(TypeError, exctype, *args[:-1])
330 # check with one argument too much
331 self.assertRaises(TypeError, exctype, *(args + ["too much"]))
332 # check with one argument of the wrong type
333 wrongargs = [ "spam", u"eggs", 42, 1.0, None ]
334 for i in xrange(len(args)):
335 for wrongarg in wrongargs:
336 if type(wrongarg) is type(args[i]):
337 continue
338 # build argument array
339 callargs = []
340 for j in xrange(len(args)):
341 if i==j:
342 callargs.append(wrongarg)
343 else:
344 callargs.append(args[i])
345 self.assertRaises(TypeError, exctype, *callargs)
346
347 # check with the correct number and type of arguments
348 exc = exctype(*args)
349 self.assertEqual(str(exc), msg)
350
351 def test_unicodeencodeerror(self):
352 self.check_exceptionobjectargs(
353 UnicodeEncodeError,
354 ["ascii", u"g\xfcrk", 1, 2, "ouch"],
355 "'ascii' codec can't encode character u'\\xfc' in position 1: ouch"
356 )
357 self.check_exceptionobjectargs(
358 UnicodeEncodeError,
359 ["ascii", u"g\xfcrk", 1, 4, "ouch"],
360 "'ascii' codec can't encode characters in position 1-3: ouch"
361 )
362 self.check_exceptionobjectargs(
363 UnicodeEncodeError,
364 ["ascii", u"\xfcx", 0, 1, "ouch"],
365 "'ascii' codec can't encode character u'\\xfc' in position 0: ouch"
366 )
367 self.check_exceptionobjectargs(
368 UnicodeEncodeError,
369 ["ascii", u"\u0100x", 0, 1, "ouch"],
370 "'ascii' codec can't encode character u'\\u0100' in position 0: ouch"
371 )
372 self.check_exceptionobjectargs(
373 UnicodeEncodeError,
374 ["ascii", u"\uffffx", 0, 1, "ouch"],
375 "'ascii' codec can't encode character u'\\uffff' in position 0: ouch"
376 )
377 if sys.maxunicode > 0xffff:
378 self.check_exceptionobjectargs(
379 UnicodeEncodeError,
380 ["ascii", u"\U00010000x", 0, 1, "ouch"],
381 "'ascii' codec can't encode character u'\\U00010000' in position 0: ouch"
382 )
383
384 def test_unicodedecodeerror(self):
385 self.check_exceptionobjectargs(
386 UnicodeDecodeError,
387 ["ascii", "g\xfcrk", 1, 2, "ouch"],
388 "'ascii' codec can't decode byte 0xfc in position 1: ouch"
389 )
390 self.check_exceptionobjectargs(
391 UnicodeDecodeError,
392 ["ascii", "g\xfcrk", 1, 3, "ouch"],
393 "'ascii' codec can't decode bytes in position 1-2: ouch"
394 )
395
396 def test_unicodetranslateerror(self):
397 self.check_exceptionobjectargs(
398 UnicodeTranslateError,
399 [u"g\xfcrk", 1, 2, "ouch"],
400 "can't translate character u'\\xfc' in position 1: ouch"
401 )
402 self.check_exceptionobjectargs(
403 UnicodeTranslateError,
404 [u"g\u0100rk", 1, 2, "ouch"],
405 "can't translate character u'\\u0100' in position 1: ouch"
406 )
407 self.check_exceptionobjectargs(
408 UnicodeTranslateError,
409 [u"g\uffffrk", 1, 2, "ouch"],
410 "can't translate character u'\\uffff' in position 1: ouch"
411 )
412 if sys.maxunicode > 0xffff:
413 self.check_exceptionobjectargs(
414 UnicodeTranslateError,
415 [u"g\U00010000rk", 1, 2, "ouch"],
416 "can't translate character u'\\U00010000' in position 1: ouch"
417 )
418 self.check_exceptionobjectargs(
419 UnicodeTranslateError,
420 [u"g\xfcrk", 1, 3, "ouch"],
421 "can't translate characters in position 1-2: ouch"
422 )
423
424 def test_badandgoodstrictexceptions(self):
425 # "strict" complains about a non-exception passed in
426 self.assertRaises(
427 TypeError,
428 codecs.strict_errors,
429 42
430 )
431 # "strict" complains about the wrong exception type
432 self.assertRaises(
433 Exception,
434 codecs.strict_errors,
435 Exception("ouch")
436 )
437
438 # If the correct exception is passed in, "strict" raises it
439 self.assertRaises(
440 UnicodeEncodeError,
441 codecs.strict_errors,
442 UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")
443 )
444
445 def test_badandgoodignoreexceptions(self):
446 # "ignore" complains about a non-exception passed in
447 self.assertRaises(
448 TypeError,
449 codecs.ignore_errors,
450 42
451 )
452 # "ignore" complains about the wrong exception type
453 self.assertRaises(
454 TypeError,
455 codecs.ignore_errors,
456 UnicodeError("ouch")
457 )
458 # If the correct exception is passed in, "ignore" returns an empty replacement
459 self.assertEqual(
460 codecs.ignore_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
461 (u"", 1)
462 )
463 self.assertEqual(
464 codecs.ignore_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
465 (u"", 1)
466 )
467 self.assertEqual(
468 codecs.ignore_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
469 (u"", 1)
470 )
471
472 def test_badandgoodreplaceexceptions(self):
473 # "replace" complains about a non-exception passed in
474 self.assertRaises(
475 TypeError,
476 codecs.replace_errors,
477 42
478 )
479 # "replace" complains about the wrong exception type
480 self.assertRaises(
481 TypeError,
482 codecs.replace_errors,
483 UnicodeError("ouch")
484 )
485 self.assertRaises(
486 TypeError,
487 codecs.replace_errors,
488 BadObjectUnicodeEncodeError()
489 )
490 self.assertRaises(
491 TypeError,
492 codecs.replace_errors,
493 BadObjectUnicodeDecodeError()
494 )
495 # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement
496 self.assertEqual(
497 codecs.replace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
498 (u"?", 1)
499 )
500 self.assertEqual(
501 codecs.replace_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
502 (u"\ufffd", 1)
503 )
504 self.assertEqual(
505 codecs.replace_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
506 (u"\ufffd", 1)
507 )
508
509 def test_badandgoodxmlcharrefreplaceexceptions(self):
510 # "xmlcharrefreplace" complains about a non-exception passed in
511 self.assertRaises(
512 TypeError,
513 codecs.xmlcharrefreplace_errors,
514 42
515 )
516 # "xmlcharrefreplace" complains about the wrong exception types
517 self.assertRaises(
518 TypeError,
519 codecs.xmlcharrefreplace_errors,
520 UnicodeError("ouch")
521 )
522 # "xmlcharrefreplace" can only be used for encoding
523 self.assertRaises(
524 TypeError,
525 codecs.xmlcharrefreplace_errors,
526 UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
527 )
528 self.assertRaises(
529 TypeError,
530 codecs.xmlcharrefreplace_errors,
531 UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
532 )
533 # Use the correct exception
534 cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 0x3042)
535 s = "".join(unichr(c) for c in cs)
536 self.assertEqual(
537 codecs.xmlcharrefreplace_errors(
538 UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
539 ),
540 (u"".join(u"&#%d;" % ord(c) for c in s), len(s))
541 )
542
543 def test_badandgoodbackslashreplaceexceptions(self):
544 # "backslashreplace" complains about a non-exception passed in
545 self.assertRaises(
546 TypeError,
547 codecs.backslashreplace_errors,
548 42
549 )
550 # "backslashreplace" complains about the wrong exception types
551 self.assertRaises(
552 TypeError,
553 codecs.backslashreplace_errors,
554 UnicodeError("ouch")
555 )
556 # "backslashreplace" can only be used for encoding
557 self.assertRaises(
558 TypeError,
559 codecs.backslashreplace_errors,
560 UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
561 )
562 self.assertRaises(
563 TypeError,
564 codecs.backslashreplace_errors,
565 UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
566 )
567 # Use the correct exception
568 self.assertEqual(
569 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
570 (u"\\u3042", 1)
571 )
572 self.assertEqual(
573 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\x00", 0, 1, "ouch")),
574 (u"\\x00", 1)
575 )
576 self.assertEqual(
577 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\xff", 0, 1, "ouch")),
578 (u"\\xff", 1)
579 )
580 self.assertEqual(
581 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u0100", 0, 1, "ouch")),
582 (u"\\u0100", 1)
583 )
584 self.assertEqual(
585 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\uffff", 0, 1, "ouch")),
586 (u"\\uffff", 1)
587 )
588 if sys.maxunicode>0xffff:
589 self.assertEqual(
590 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U00010000", 0, 1, "ouch")),
591 (u"\\U00010000", 1)
592 )
593 self.assertEqual(
594 codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U0010ffff", 0, 1, "ouch")),
595 (u"\\U0010ffff", 1)
596 )
597
598 def test_badhandlerresults(self):
599 results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
600 encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15")
601
602 for res in results:
603 codecs.register_error("test.badhandler", lambda x: res)
604 for enc in encs:
605 self.assertRaises(
606 TypeError,
607 u"\u3042".encode,
608 enc,
609 "test.badhandler"
610 )
611 for (enc, bytes) in (
612 ("ascii", "\xff"),
613 ("utf-8", "\xff"),
614 ("utf-7", "+x-"),
615 ("unicode-internal", "\x00"),
616 ):
617 self.assertRaises(
618 TypeError,
619 bytes.decode,
620 enc,
621 "test.badhandler"
622 )
623
624 def test_lookup(self):
625 self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict"))
626 self.assertEqual(codecs.ignore_errors, codecs.lookup_error("ignore"))
627 self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict"))
628 self.assertEqual(
629 codecs.xmlcharrefreplace_errors,
630 codecs.lookup_error("xmlcharrefreplace")
631 )
632 self.assertEqual(
633 codecs.backslashreplace_errors,
634 codecs.lookup_error("backslashreplace")
635 )
636
637 def test_unencodablereplacement(self):
638 def unencrepl(exc):
639 if isinstance(exc, UnicodeEncodeError):
640 return (u"\u4242", exc.end)
641 else:
642 raise TypeError("don't know how to handle %r" % exc)
643 codecs.register_error("test.unencreplhandler", unencrepl)
644 for enc in ("ascii", "iso-8859-1", "iso-8859-15"):
645 self.assertRaises(
646 UnicodeEncodeError,
647 u"\u4242".encode,
648 enc,
649 "test.unencreplhandler"
650 )
651
652 def test_badregistercall(self):
653 # enhance coverage of:
654 # Modules/_codecsmodule.c::register_error()
655 # Python/codecs.c::PyCodec_RegisterError()
656 self.assertRaises(TypeError, codecs.register_error, 42)
657 self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42)
658
659 def test_badlookupcall(self):
660 # enhance coverage of:
661 # Modules/_codecsmodule.c::lookup_error()
662 self.assertRaises(TypeError, codecs.lookup_error)
663
664 def test_unknownhandler(self):
665 # enhance coverage of:
666 # Modules/_codecsmodule.c::lookup_error()
667 self.assertRaises(LookupError, codecs.lookup_error, "test.unknown")
668
669 def test_xmlcharrefvalues(self):
670 # enhance coverage of:
671 # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
672 # and inline implementations
673 v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000)
674 if sys.maxunicode>=100000:
675 v += (100000, 500000, 1000000)
676 s = u"".join([unichr(x) for x in v])
677 codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
678 for enc in ("ascii", "iso-8859-15"):
679 for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"):
680 s.encode(enc, err)
681
682 def test_decodehelper(self):
683 # enhance coverage of:
684 # Objects/unicodeobject.c::unicode_decode_call_errorhandler()
685 # and callers
686 self.assertRaises(LookupError, "\xff".decode, "ascii", "test.unknown")
687
688 def baddecodereturn1(exc):
689 return 42
690 codecs.register_error("test.baddecodereturn1", baddecodereturn1)
691 self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1")
692 self.assertRaises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1")
693 self.assertRaises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1")
694 self.assertRaises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
695 self.assertRaises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
696 self.assertRaises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
697
698 def baddecodereturn2(exc):
699 return (u"?", None)
700 codecs.register_error("test.baddecodereturn2", baddecodereturn2)
701 self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn2")
702
703 handler = PosReturn()
704 codecs.register_error("test.posreturn", handler.handle)
705
706 # Valid negative position
707 handler.pos = -1
708 self.assertEqual("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
709
710 # Valid negative position
711 handler.pos = -2
712 self.assertEqual("\xff0".decode("ascii", "test.posreturn"), u"<?><?>")
713
714 # Negative position out of bounds
715 handler.pos = -3
716 self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
717
718 # Valid positive position
719 handler.pos = 1
720 self.assertEqual("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
721
722 # Largest valid positive position (one beyond end of input)
723 handler.pos = 2
724 self.assertEqual("\xff0".decode("ascii", "test.posreturn"), u"<?>")
725
726 # Invalid positive position
727 handler.pos = 3
728 self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
729
730 # Restart at the "0"
731 handler.pos = 6
732 self.assertEqual("\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), u"<?>0")
733
734 class D(dict):
735 def __getitem__(self, key):
736 raise ValueError
737 self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
738 self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
739 self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: 0x110000})
740
741 def test_encodehelper(self):
742 # enhance coverage of:
743 # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
744 # and callers
745 self.assertRaises(LookupError, u"\xff".encode, "ascii", "test.unknown")
746
747 def badencodereturn1(exc):
748 return 42
749 codecs.register_error("test.badencodereturn1", badencodereturn1)
750 self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn1")
751
752 def badencodereturn2(exc):
753 return (u"?", None)
754 codecs.register_error("test.badencodereturn2", badencodereturn2)
755 self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn2")
756
757 handler = PosReturn()
758 codecs.register_error("test.posreturn", handler.handle)
759
760 # Valid negative position
761 handler.pos = -1
762 self.assertEqual(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
763
764 # Valid negative position
765 handler.pos = -2
766 self.assertEqual(u"\xff0".encode("ascii", "test.posreturn"), "<?><?>")
767
768 # Negative position out of bounds
769 handler.pos = -3
770 self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
771
772 # Valid positive position
773 handler.pos = 1
774 self.assertEqual(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
775
776 # Largest valid positive position (one beyond end of input
777 handler.pos = 2
778 self.assertEqual(u"\xff0".encode("ascii", "test.posreturn"), "<?>")
779
780 # Invalid positive position
781 handler.pos = 3
782 self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
783
784 handler.pos = 0
785
786 class D(dict):
787 def __getitem__(self, key):
788 raise ValueError
789 for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.posreturn"):
790 self.assertRaises(UnicodeError, codecs.charmap_encode, u"\xff", err, {0xff: None})
791 self.assertRaises(ValueError, codecs.charmap_encode, u"\xff", err, D())
792 self.assertRaises(TypeError, codecs.charmap_encode, u"\xff", err, {0xff: 300})
793
794 def test_translatehelper(self):
795 # enhance coverage of:
796 # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
797 # and callers
798 # (Unfortunately the errors argument is not directly accessible
799 # from Python, so we can't test that much)
800 class D(dict):
801 def __getitem__(self, key):
802 raise ValueError
803 self.assertRaises(ValueError, u"\xff".translate, D())
804 self.assertRaises(TypeError, u"\xff".translate, {0xff: sys.maxunicode+1})
805 self.assertRaises(TypeError, u"\xff".translate, {0xff: ()})
806
807 def test_bug828737(self):
808 charmap = {
809 ord("&"): u"&amp;",
810 ord("<"): u"&lt;",
811 ord(">"): u"&gt;",
812 ord('"'): u"&quot;",
813 }
814
815 for n in (1, 10, 100, 1000):
816 text = u'abc<def>ghi'*n
817 text.translate(charmap)
818
819def test_main():
820 test.test_support.run_unittest(CodecCallbackTest)
821
822if __name__ == "__main__":
823 test_main()
Note: See TracBrowser for help on using the repository browser.