source: python/trunk/Lib/email/charset.py

Last change on this file was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 15.7 KB
RevLine 
[2]1# Copyright (C) 2001-2006 Python Software Foundation
2# Author: Ben Gertzfield, Barry Warsaw
3# Contact: email-sig@python.org
4
5__all__ = [
6 'Charset',
7 'add_alias',
8 'add_charset',
9 'add_codec',
10 ]
11
[391]12import codecs
[2]13import email.base64mime
14import email.quoprimime
15
16from email import errors
17from email.encoders import encode_7or8bit
18
19
20
21
22# Flags for types of header encodings
23QP = 1 # Quoted-Printable
24BASE64 = 2 # Base64
25SHORTEST = 3 # the shorter of QP and base64, but only for headers
26
27# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
28MISC_LEN = 7
29
30DEFAULT_CHARSET = 'us-ascii'
31
32
33
34
35# Defaults
36CHARSETS = {
37 # input header enc body enc output conv
38 'iso-8859-1': (QP, QP, None),
39 'iso-8859-2': (QP, QP, None),
40 'iso-8859-3': (QP, QP, None),
41 'iso-8859-4': (QP, QP, None),
42 # iso-8859-5 is Cyrillic, and not especially used
43 # iso-8859-6 is Arabic, also not particularly used
44 # iso-8859-7 is Greek, QP will not make it readable
45 # iso-8859-8 is Hebrew, QP will not make it readable
46 'iso-8859-9': (QP, QP, None),
47 'iso-8859-10': (QP, QP, None),
48 # iso-8859-11 is Thai, QP will not make it readable
49 'iso-8859-13': (QP, QP, None),
50 'iso-8859-14': (QP, QP, None),
51 'iso-8859-15': (QP, QP, None),
52 'iso-8859-16': (QP, QP, None),
53 'windows-1252':(QP, QP, None),
54 'viscii': (QP, QP, None),
55 'us-ascii': (None, None, None),
56 'big5': (BASE64, BASE64, None),
57 'gb2312': (BASE64, BASE64, None),
58 'euc-jp': (BASE64, None, 'iso-2022-jp'),
59 'shift_jis': (BASE64, None, 'iso-2022-jp'),
60 'iso-2022-jp': (BASE64, None, None),
61 'koi8-r': (BASE64, BASE64, None),
62 'utf-8': (SHORTEST, BASE64, 'utf-8'),
63 # We're making this one up to represent raw unencoded 8-bit
64 '8bit': (None, BASE64, 'utf-8'),
65 }
66
67# Aliases for other commonly-used names for character sets. Map
68# them to the real ones used in email.
69ALIASES = {
70 'latin_1': 'iso-8859-1',
71 'latin-1': 'iso-8859-1',
72 'latin_2': 'iso-8859-2',
73 'latin-2': 'iso-8859-2',
74 'latin_3': 'iso-8859-3',
75 'latin-3': 'iso-8859-3',
76 'latin_4': 'iso-8859-4',
77 'latin-4': 'iso-8859-4',
78 'latin_5': 'iso-8859-9',
79 'latin-5': 'iso-8859-9',
80 'latin_6': 'iso-8859-10',
81 'latin-6': 'iso-8859-10',
82 'latin_7': 'iso-8859-13',
83 'latin-7': 'iso-8859-13',
84 'latin_8': 'iso-8859-14',
85 'latin-8': 'iso-8859-14',
86 'latin_9': 'iso-8859-15',
87 'latin-9': 'iso-8859-15',
88 'latin_10':'iso-8859-16',
89 'latin-10':'iso-8859-16',
90 'cp949': 'ks_c_5601-1987',
91 'euc_jp': 'euc-jp',
92 'euc_kr': 'euc-kr',
93 'ascii': 'us-ascii',
94 }
95
96
97# Map charsets to their Unicode codec strings.
98CODEC_MAP = {
99 'gb2312': 'eucgb2312_cn',
100 'big5': 'big5_tw',
101 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
102 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
103 # Let that stuff pass through without conversion to/from Unicode.
104 'us-ascii': None,
105 }
106
107
108
109
110# Convenience functions for extending the above mappings
111def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
112 """Add character set properties to the global registry.
113
114 charset is the input character set, and must be the canonical name of a
115 character set.
116
117 Optional header_enc and body_enc is either Charset.QP for
118 quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
119 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
120 is only valid for header_enc. It describes how message headers and
121 message bodies in the input charset are to be encoded. Default is no
122 encoding.
123
124 Optional output_charset is the character set that the output should be
125 in. Conversions will proceed from input charset, to Unicode, to the
126 output charset when the method Charset.convert() is called. The default
127 is to output in the same character set as the input.
128
129 Both input_charset and output_charset must have Unicode codec entries in
130 the module's charset-to-codec mapping; use add_codec(charset, codecname)
131 to add codecs the module does not know about. See the codecs module's
132 documentation for more information.
133 """
134 if body_enc == SHORTEST:
135 raise ValueError('SHORTEST not allowed for body_enc')
136 CHARSETS[charset] = (header_enc, body_enc, output_charset)
137
138
139def add_alias(alias, canonical):
140 """Add a character set alias.
141
142 alias is the alias name, e.g. latin-1
143 canonical is the character set's canonical name, e.g. iso-8859-1
144 """
145 ALIASES[alias] = canonical
146
147
148def add_codec(charset, codecname):
149 """Add a codec that map characters in the given charset to/from Unicode.
150
151 charset is the canonical name of a character set. codecname is the name
152 of a Python codec, as appropriate for the second argument to the unicode()
153 built-in, or to the encode() method of a Unicode string.
154 """
155 CODEC_MAP[charset] = codecname
156
157
158
159
160class Charset:
161 """Map character sets to their email properties.
162
163 This class provides information about the requirements imposed on email
164 for a specific character set. It also provides convenience routines for
165 converting between character sets, given the availability of the
166 applicable codecs. Given a character set, it will do its best to provide
167 information on how to use that character set in an email in an
168 RFC-compliant way.
169
170 Certain character sets must be encoded with quoted-printable or base64
171 when used in email headers or bodies. Certain character sets must be
172 converted outright, and are not allowed in email. Instances of this
173 module expose the following information about a character set:
174
175 input_charset: The initial character set specified. Common aliases
176 are converted to their `official' email names (e.g. latin_1
177 is converted to iso-8859-1). Defaults to 7-bit us-ascii.
178
179 header_encoding: If the character set must be encoded before it can be
180 used in an email header, this attribute will be set to
181 Charset.QP (for quoted-printable), Charset.BASE64 (for
182 base64 encoding), or Charset.SHORTEST for the shortest of
183 QP or BASE64 encoding. Otherwise, it will be None.
184
185 body_encoding: Same as header_encoding, but describes the encoding for the
[391]186 mail message's body, which indeed may be different than the
[2]187 header encoding. Charset.SHORTEST is not allowed for
188 body_encoding.
189
190 output_charset: Some character sets must be converted before they can be
191 used in email headers or bodies. If the input_charset is
192 one of them, this attribute will contain the name of the
193 charset output will be converted to. Otherwise, it will
194 be None.
195
196 input_codec: The name of the Python codec used to convert the
197 input_charset to Unicode. If no conversion codec is
198 necessary, this attribute will be None.
199
200 output_codec: The name of the Python codec used to convert Unicode
201 to the output_charset. If no conversion codec is necessary,
202 this attribute will have the same value as the input_codec.
203 """
204 def __init__(self, input_charset=DEFAULT_CHARSET):
205 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
206 # unicode because its .lower() is locale insensitive. If the argument
207 # is already a unicode, we leave it at that, but ensure that the
208 # charset is ASCII, as the standard (RFC XXX) requires.
209 try:
210 if isinstance(input_charset, unicode):
211 input_charset.encode('ascii')
[391]212 else:
213 input_charset = unicode(input_charset, 'ascii')
214 except UnicodeError:
215 raise errors.CharsetError(input_charset)
216 input_charset = input_charset.lower().encode('ascii')
217 # Set the input charset after filtering through the aliases and/or codecs
218 if not (input_charset in ALIASES or input_charset in CHARSETS):
[2]219 try:
220 input_charset = codecs.lookup(input_charset).name
221 except LookupError:
222 pass
223 self.input_charset = ALIASES.get(input_charset, input_charset)
224 # We can try to guess which encoding and conversion to use by the
225 # charset_map dictionary. Try that first, but let the user override
226 # it.
227 henc, benc, conv = CHARSETS.get(self.input_charset,
228 (SHORTEST, BASE64, None))
229 if not conv:
230 conv = self.input_charset
231 # Set the attributes, allowing the arguments to override the default.
232 self.header_encoding = henc
233 self.body_encoding = benc
234 self.output_charset = ALIASES.get(conv, conv)
235 # Now set the codecs. If one isn't defined for input_charset,
236 # guess and try a Unicode codec with the same name as input_codec.
237 self.input_codec = CODEC_MAP.get(self.input_charset,
238 self.input_charset)
239 self.output_codec = CODEC_MAP.get(self.output_charset,
240 self.output_charset)
241
242 def __str__(self):
243 return self.input_charset.lower()
244
245 __repr__ = __str__
246
247 def __eq__(self, other):
248 return str(self) == str(other).lower()
249
250 def __ne__(self, other):
251 return not self.__eq__(other)
252
253 def get_body_encoding(self):
254 """Return the content-transfer-encoding used for body encoding.
255
256 This is either the string `quoted-printable' or `base64' depending on
257 the encoding used, or it is a function in which case you should call
258 the function with a single argument, the Message object being
259 encoded. The function should then set the Content-Transfer-Encoding
260 header itself to whatever is appropriate.
261
262 Returns "quoted-printable" if self.body_encoding is QP.
263 Returns "base64" if self.body_encoding is BASE64.
264 Returns "7bit" otherwise.
265 """
266 assert self.body_encoding != SHORTEST
267 if self.body_encoding == QP:
268 return 'quoted-printable'
269 elif self.body_encoding == BASE64:
270 return 'base64'
271 else:
272 return encode_7or8bit
273
274 def convert(self, s):
275 """Convert a string from the input_codec to the output_codec."""
276 if self.input_codec != self.output_codec:
277 return unicode(s, self.input_codec).encode(self.output_codec)
278 else:
279 return s
280
281 def to_splittable(self, s):
282 """Convert a possibly multibyte string to a safely splittable format.
283
284 Uses the input_codec to try and convert the string to Unicode, so it
285 can be safely split on character boundaries (even for multibyte
286 characters).
287
288 Returns the string as-is if it isn't known how to convert it to
289 Unicode with the input_charset.
290
291 Characters that could not be converted to Unicode will be replaced
292 with the Unicode replacement character U+FFFD.
293 """
294 if isinstance(s, unicode) or self.input_codec is None:
295 return s
296 try:
297 return unicode(s, self.input_codec, 'replace')
298 except LookupError:
299 # Input codec not installed on system, so return the original
300 # string unchanged.
301 return s
302
303 def from_splittable(self, ustr, to_output=True):
304 """Convert a splittable string back into an encoded string.
305
306 Uses the proper codec to try and convert the string from Unicode back
307 into an encoded format. Return the string as-is if it is not Unicode,
308 or if it could not be converted from Unicode.
309
310 Characters that could not be converted from Unicode will be replaced
311 with an appropriate character (usually '?').
312
313 If to_output is True (the default), uses output_codec to convert to an
314 encoded format. If to_output is False, uses input_codec.
315 """
316 if to_output:
317 codec = self.output_codec
318 else:
319 codec = self.input_codec
320 if not isinstance(ustr, unicode) or codec is None:
321 return ustr
322 try:
323 return ustr.encode(codec, 'replace')
324 except LookupError:
325 # Output codec not installed
326 return ustr
327
328 def get_output_charset(self):
329 """Return the output character set.
330
331 This is self.output_charset if that is not None, otherwise it is
332 self.input_charset.
333 """
334 return self.output_charset or self.input_charset
335
336 def encoded_header_len(self, s):
337 """Return the length of the encoded header string."""
338 cset = self.get_output_charset()
339 # The len(s) of a 7bit encoding is len(s)
340 if self.header_encoding == BASE64:
341 return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
342 elif self.header_encoding == QP:
343 return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
344 elif self.header_encoding == SHORTEST:
345 lenb64 = email.base64mime.base64_len(s)
346 lenqp = email.quoprimime.header_quopri_len(s)
347 return min(lenb64, lenqp) + len(cset) + MISC_LEN
348 else:
349 return len(s)
350
351 def header_encode(self, s, convert=False):
352 """Header-encode a string, optionally converting it to output_charset.
353
354 If convert is True, the string will be converted from the input
355 charset to the output charset automatically. This is not useful for
356 multibyte character sets, which have line length issues (multibyte
357 characters must be split on a character, not a byte boundary); use the
358 high-level Header class to deal with these issues. convert defaults
359 to False.
360
361 The type of encoding (base64 or quoted-printable) will be based on
362 self.header_encoding.
363 """
364 cset = self.get_output_charset()
365 if convert:
366 s = self.convert(s)
367 # 7bit/8bit encodings return the string unchanged (modulo conversions)
368 if self.header_encoding == BASE64:
369 return email.base64mime.header_encode(s, cset)
370 elif self.header_encoding == QP:
371 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
372 elif self.header_encoding == SHORTEST:
373 lenb64 = email.base64mime.base64_len(s)
374 lenqp = email.quoprimime.header_quopri_len(s)
375 if lenb64 < lenqp:
376 return email.base64mime.header_encode(s, cset)
377 else:
378 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
379 else:
380 return s
381
382 def body_encode(self, s, convert=True):
383 """Body-encode a string and convert it to output_charset.
384
385 If convert is True (the default), the string will be converted from
386 the input charset to output charset automatically. Unlike
387 header_encode(), there are no issues with byte boundaries and
388 multibyte charsets in email bodies, so this is usually pretty safe.
389
390 The type of encoding (base64 or quoted-printable) will be based on
391 self.body_encoding.
392 """
393 if convert:
394 s = self.convert(s)
395 # 7bit/8bit encodings return the string unchanged (module conversions)
396 if self.body_encoding is BASE64:
397 return email.base64mime.body_encode(s)
398 elif self.body_encoding is QP:
399 return email.quoprimime.body_encode(s)
400 else:
401 return s
Note: See TracBrowser for help on using the repository browser.