source: python/vendor/Python-2.6.5/Lib/email/charset.py

Last change on this file was 2, checked in by Yuri Dario, 15 years ago

Initial import for vendor code.
  • Property svn:eol-style set to native
File size: 15.4 KB
Line 
1# Copyright (C) 2001-2006 Python Software Foundation
2# Author: Ben Gertzfield, Barry Warsaw
3# Contact: email-sig@python.org
4
5__all__ = [
6 'Charset',
7 'add_alias',
8 'add_charset',
9 'add_codec',
10 ]
11
12import email.base64mime
13import email.quoprimime
14
15from email import errors
16from email.encoders import encode_7or8bit
17
18
19
20
21# Flags for types of header encodings
22QP = 1 # Quoted-Printable
23BASE64 = 2 # Base64
24SHORTEST = 3 # the shorter of QP and base64, but only for headers
25
26# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
27MISC_LEN = 7
28
29DEFAULT_CHARSET = 'us-ascii'
30
31
32
33
34# Defaults
35CHARSETS = {
36 # input header enc body enc output conv
37 'iso-8859-1': (QP, QP, None),
38 'iso-8859-2': (QP, QP, None),
39 'iso-8859-3': (QP, QP, None),
40 'iso-8859-4': (QP, QP, None),
41 # iso-8859-5 is Cyrillic, and not especially used
42 # iso-8859-6 is Arabic, also not particularly used
43 # iso-8859-7 is Greek, QP will not make it readable
44 # iso-8859-8 is Hebrew, QP will not make it readable
45 'iso-8859-9': (QP, QP, None),
46 'iso-8859-10': (QP, QP, None),
47 # iso-8859-11 is Thai, QP will not make it readable
48 'iso-8859-13': (QP, QP, None),
49 'iso-8859-14': (QP, QP, None),
50 'iso-8859-15': (QP, QP, None),
51 'iso-8859-16': (QP, QP, None),
52 'windows-1252':(QP, QP, None),
53 'viscii': (QP, QP, None),
54 'us-ascii': (None, None, None),
55 'big5': (BASE64, BASE64, None),
56 'gb2312': (BASE64, BASE64, None),
57 'euc-jp': (BASE64, None, 'iso-2022-jp'),
58 'shift_jis': (BASE64, None, 'iso-2022-jp'),
59 'iso-2022-jp': (BASE64, None, None),
60 'koi8-r': (BASE64, BASE64, None),
61 'utf-8': (SHORTEST, BASE64, 'utf-8'),
62 # We're making this one up to represent raw unencoded 8-bit
63 '8bit': (None, BASE64, 'utf-8'),
64 }
65
66# Aliases for other commonly-used names for character sets. Map
67# them to the real ones used in email.
68ALIASES = {
69 'latin_1': 'iso-8859-1',
70 'latin-1': 'iso-8859-1',
71 'latin_2': 'iso-8859-2',
72 'latin-2': 'iso-8859-2',
73 'latin_3': 'iso-8859-3',
74 'latin-3': 'iso-8859-3',
75 'latin_4': 'iso-8859-4',
76 'latin-4': 'iso-8859-4',
77 'latin_5': 'iso-8859-9',
78 'latin-5': 'iso-8859-9',
79 'latin_6': 'iso-8859-10',
80 'latin-6': 'iso-8859-10',
81 'latin_7': 'iso-8859-13',
82 'latin-7': 'iso-8859-13',
83 'latin_8': 'iso-8859-14',
84 'latin-8': 'iso-8859-14',
85 'latin_9': 'iso-8859-15',
86 'latin-9': 'iso-8859-15',
87 'latin_10':'iso-8859-16',
88 'latin-10':'iso-8859-16',
89 'cp949': 'ks_c_5601-1987',
90 'euc_jp': 'euc-jp',
91 'euc_kr': 'euc-kr',
92 'ascii': 'us-ascii',
93 }
94
95
96# Map charsets to their Unicode codec strings.
97CODEC_MAP = {
98 'gb2312': 'eucgb2312_cn',
99 'big5': 'big5_tw',
100 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
101 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
102 # Let that stuff pass through without conversion to/from Unicode.
103 'us-ascii': None,
104 }
105
106
107
108
109# Convenience functions for extending the above mappings
110def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
111 """Add character set properties to the global registry.
112
113 charset is the input character set, and must be the canonical name of a
114 character set.
115
116 Optional header_enc and body_enc is either Charset.QP for
117 quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
118 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
119 is only valid for header_enc. It describes how message headers and
120 message bodies in the input charset are to be encoded. Default is no
121 encoding.
122
123 Optional output_charset is the character set that the output should be
124 in. Conversions will proceed from input charset, to Unicode, to the
125 output charset when the method Charset.convert() is called. The default
126 is to output in the same character set as the input.
127
128 Both input_charset and output_charset must have Unicode codec entries in
129 the module's charset-to-codec mapping; use add_codec(charset, codecname)
130 to add codecs the module does not know about. See the codecs module's
131 documentation for more information.
132 """
133 if body_enc == SHORTEST:
134 raise ValueError('SHORTEST not allowed for body_enc')
135 CHARSETS[charset] = (header_enc, body_enc, output_charset)
136
137
138def add_alias(alias, canonical):
139 """Add a character set alias.
140
141 alias is the alias name, e.g. latin-1
142 canonical is the character set's canonical name, e.g. iso-8859-1
143 """
144 ALIASES[alias] = canonical
145
146
147def add_codec(charset, codecname):
148 """Add a codec that map characters in the given charset to/from Unicode.
149
150 charset is the canonical name of a character set. codecname is the name
151 of a Python codec, as appropriate for the second argument to the unicode()
152 built-in, or to the encode() method of a Unicode string.
153 """
154 CODEC_MAP[charset] = codecname
155
156
157
158
159class Charset:
160 """Map character sets to their email properties.
161
162 This class provides information about the requirements imposed on email
163 for a specific character set. It also provides convenience routines for
164 converting between character sets, given the availability of the
165 applicable codecs. Given a character set, it will do its best to provide
166 information on how to use that character set in an email in an
167 RFC-compliant way.
168
169 Certain character sets must be encoded with quoted-printable or base64
170 when used in email headers or bodies. Certain character sets must be
171 converted outright, and are not allowed in email. Instances of this
172 module expose the following information about a character set:
173
174 input_charset: The initial character set specified. Common aliases
175 are converted to their `official' email names (e.g. latin_1
176 is converted to iso-8859-1). Defaults to 7-bit us-ascii.
177
178 header_encoding: If the character set must be encoded before it can be
179 used in an email header, this attribute will be set to
180 Charset.QP (for quoted-printable), Charset.BASE64 (for
181 base64 encoding), or Charset.SHORTEST for the shortest of
182 QP or BASE64 encoding. Otherwise, it will be None.
183
184 body_encoding: Same as header_encoding, but describes the encoding for the
185 mail message's body, which indeed may be different than the
186 header encoding. Charset.SHORTEST is not allowed for
187 body_encoding.
188
189 output_charset: Some character sets must be converted before the can be
190 used in email headers or bodies. If the input_charset is
191 one of them, this attribute will contain the name of the
192 charset output will be converted to. Otherwise, it will
193 be None.
194
195 input_codec: The name of the Python codec used to convert the
196 input_charset to Unicode. If no conversion codec is
197 necessary, this attribute will be None.
198
199 output_codec: The name of the Python codec used to convert Unicode
200 to the output_charset. If no conversion codec is necessary,
201 this attribute will have the same value as the input_codec.
202 """
203 def __init__(self, input_charset=DEFAULT_CHARSET):
204 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
205 # unicode because its .lower() is locale insensitive. If the argument
206 # is already a unicode, we leave it at that, but ensure that the
207 # charset is ASCII, as the standard (RFC XXX) requires.
208 try:
209 if isinstance(input_charset, unicode):
210 input_charset.encode('ascii')
211 else:
212 input_charset = unicode(input_charset, 'ascii')
213 except UnicodeError:
214 raise errors.CharsetError(input_charset)
215 input_charset = input_charset.lower()
216 # Set the input charset after filtering through the aliases
217 self.input_charset = ALIASES.get(input_charset, input_charset)
218 # We can try to guess which encoding and conversion to use by the
219 # charset_map dictionary. Try that first, but let the user override
220 # it.
221 henc, benc, conv = CHARSETS.get(self.input_charset,
222 (SHORTEST, BASE64, None))
223 if not conv:
224 conv = self.input_charset
225 # Set the attributes, allowing the arguments to override the default.
226 self.header_encoding = henc
227 self.body_encoding = benc
228 self.output_charset = ALIASES.get(conv, conv)
229 # Now set the codecs. If one isn't defined for input_charset,
230 # guess and try a Unicode codec with the same name as input_codec.
231 self.input_codec = CODEC_MAP.get(self.input_charset,
232 self.input_charset)
233 self.output_codec = CODEC_MAP.get(self.output_charset,
234 self.output_charset)
235
236 def __str__(self):
237 return self.input_charset.lower()
238
239 __repr__ = __str__
240
241 def __eq__(self, other):
242 return str(self) == str(other).lower()
243
244 def __ne__(self, other):
245 return not self.__eq__(other)
246
247 def get_body_encoding(self):
248 """Return the content-transfer-encoding used for body encoding.
249
250 This is either the string `quoted-printable' or `base64' depending on
251 the encoding used, or it is a function in which case you should call
252 the function with a single argument, the Message object being
253 encoded. The function should then set the Content-Transfer-Encoding
254 header itself to whatever is appropriate.
255
256 Returns "quoted-printable" if self.body_encoding is QP.
257 Returns "base64" if self.body_encoding is BASE64.
258 Returns "7bit" otherwise.
259 """
260 assert self.body_encoding != SHORTEST
261 if self.body_encoding == QP:
262 return 'quoted-printable'
263 elif self.body_encoding == BASE64:
264 return 'base64'
265 else:
266 return encode_7or8bit
267
268 def convert(self, s):
269 """Convert a string from the input_codec to the output_codec."""
270 if self.input_codec != self.output_codec:
271 return unicode(s, self.input_codec).encode(self.output_codec)
272 else:
273 return s
274
275 def to_splittable(self, s):
276 """Convert a possibly multibyte string to a safely splittable format.
277
278 Uses the input_codec to try and convert the string to Unicode, so it
279 can be safely split on character boundaries (even for multibyte
280 characters).
281
282 Returns the string as-is if it isn't known how to convert it to
283 Unicode with the input_charset.
284
285 Characters that could not be converted to Unicode will be replaced
286 with the Unicode replacement character U+FFFD.
287 """
288 if isinstance(s, unicode) or self.input_codec is None:
289 return s
290 try:
291 return unicode(s, self.input_codec, 'replace')
292 except LookupError:
293 # Input codec not installed on system, so return the original
294 # string unchanged.
295 return s
296
297 def from_splittable(self, ustr, to_output=True):
298 """Convert a splittable string back into an encoded string.
299
300 Uses the proper codec to try and convert the string from Unicode back
301 into an encoded format. Return the string as-is if it is not Unicode,
302 or if it could not be converted from Unicode.
303
304 Characters that could not be converted from Unicode will be replaced
305 with an appropriate character (usually '?').
306
307 If to_output is True (the default), uses output_codec to convert to an
308 encoded format. If to_output is False, uses input_codec.
309 """
310 if to_output:
311 codec = self.output_codec
312 else:
313 codec = self.input_codec
314 if not isinstance(ustr, unicode) or codec is None:
315 return ustr
316 try:
317 return ustr.encode(codec, 'replace')
318 except LookupError:
319 # Output codec not installed
320 return ustr
321
322 def get_output_charset(self):
323 """Return the output character set.
324
325 This is self.output_charset if that is not None, otherwise it is
326 self.input_charset.
327 """
328 return self.output_charset or self.input_charset
329
330 def encoded_header_len(self, s):
331 """Return the length of the encoded header string."""
332 cset = self.get_output_charset()
333 # The len(s) of a 7bit encoding is len(s)
334 if self.header_encoding == BASE64:
335 return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
336 elif self.header_encoding == QP:
337 return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
338 elif self.header_encoding == SHORTEST:
339 lenb64 = email.base64mime.base64_len(s)
340 lenqp = email.quoprimime.header_quopri_len(s)
341 return min(lenb64, lenqp) + len(cset) + MISC_LEN
342 else:
343 return len(s)
344
345 def header_encode(self, s, convert=False):
346 """Header-encode a string, optionally converting it to output_charset.
347
348 If convert is True, the string will be converted from the input
349 charset to the output charset automatically. This is not useful for
350 multibyte character sets, which have line length issues (multibyte
351 characters must be split on a character, not a byte boundary); use the
352 high-level Header class to deal with these issues. convert defaults
353 to False.
354
355 The type of encoding (base64 or quoted-printable) will be based on
356 self.header_encoding.
357 """
358 cset = self.get_output_charset()
359 if convert:
360 s = self.convert(s)
361 # 7bit/8bit encodings return the string unchanged (modulo conversions)
362 if self.header_encoding == BASE64:
363 return email.base64mime.header_encode(s, cset)
364 elif self.header_encoding == QP:
365 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
366 elif self.header_encoding == SHORTEST:
367 lenb64 = email.base64mime.base64_len(s)
368 lenqp = email.quoprimime.header_quopri_len(s)
369 if lenb64 < lenqp:
370 return email.base64mime.header_encode(s, cset)
371 else:
372 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
373 else:
374 return s
375
376 def body_encode(self, s, convert=True):
377 """Body-encode a string and convert it to output_charset.
378
379 If convert is True (the default), the string will be converted from
380 the input charset to output charset automatically. Unlike
381 header_encode(), there are no issues with byte boundaries and
382 multibyte charsets in email bodies, so this is usually pretty safe.
383
384 The type of encoding (base64 or quoted-printable) will be based on
385 self.body_encoding.
386 """
387 if convert:
388 s = self.convert(s)
389 # 7bit/8bit encodings return the string unchanged (module conversions)
390 if self.body_encoding is BASE64:
391 return email.base64mime.body_encode(s)
392 elif self.body_encoding is QP:
393 return email.quoprimime.body_encode(s)
394 else:
395 return s
Note: See TracBrowser for help on using the repository browser.