source: python/trunk/Modules/cjkcodecs/_codecs_cn.c

Last change on this file was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 9.1 KB
Line 
1/*
2 * _codecs_cn.c: Codecs collection for Mainland Chinese encodings
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7#include "cjkcodecs.h"
8#include "mappings_cn.h"
9
10/**
11 * hz is predefined as 100 on AIX. So we undefine it to avoid
12 * conflict against hz codec's.
13 */
14#ifdef _AIX
15#undef hz
16#endif
17
18/* GBK and GB2312 map differently in few codepoints that are listed below:
19 *
20 * gb2312 gbk
21 * A1A4 U+30FB KATAKANA MIDDLE DOT U+00B7 MIDDLE DOT
22 * A1AA U+2015 HORIZONTAL BAR U+2014 EM DASH
23 * A844 undefined U+2015 HORIZONTAL BAR
24 */
25
26#define GBK_DECODE(dc1, dc2, assi) \
27 if ((dc1) == 0xa1 && (dc2) == 0xaa) (assi) = 0x2014; \
28 else if ((dc1) == 0xa8 && (dc2) == 0x44) (assi) = 0x2015; \
29 else if ((dc1) == 0xa1 && (dc2) == 0xa4) (assi) = 0x00b7; \
30 else TRYMAP_DEC(gb2312, assi, dc1 ^ 0x80, dc2 ^ 0x80); \
31 else TRYMAP_DEC(gbkext, assi, dc1, dc2);
32
33#define GBK_ENCODE(code, assi) \
34 if ((code) == 0x2014) (assi) = 0xa1aa; \
35 else if ((code) == 0x2015) (assi) = 0xa844; \
36 else if ((code) == 0x00b7) (assi) = 0xa1a4; \
37 else if ((code) != 0x30fb && TRYMAP_ENC_COND(gbcommon, assi, code));
38
39/*
40 * GB2312 codec
41 */
42
43ENCODER(gb2312)
44{
45 while (inleft > 0) {
46 Py_UNICODE c = IN1;
47 DBCHAR code;
48
49 if (c < 0x80) {
50 WRITE1((unsigned char)c)
51 NEXT(1, 1)
52 continue;
53 }
54 UCS4INVALID(c)
55
56 REQUIRE_OUTBUF(2)
57 TRYMAP_ENC(gbcommon, code, c);
58 else return 1;
59
60 if (code & 0x8000) /* MSB set: GBK */
61 return 1;
62
63 OUT1((code >> 8) | 0x80)
64 OUT2((code & 0xFF) | 0x80)
65 NEXT(1, 2)
66 }
67
68 return 0;
69}
70
71DECODER(gb2312)
72{
73 while (inleft > 0) {
74 unsigned char c = **inbuf;
75
76 REQUIRE_OUTBUF(1)
77
78 if (c < 0x80) {
79 OUT1(c)
80 NEXT(1, 1)
81 continue;
82 }
83
84 REQUIRE_INBUF(2)
85 TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, IN2 ^ 0x80) {
86 NEXT(2, 1)
87 }
88 else return 2;
89 }
90
91 return 0;
92}
93
94
95/*
96 * GBK codec
97 */
98
99ENCODER(gbk)
100{
101 while (inleft > 0) {
102 Py_UNICODE c = IN1;
103 DBCHAR code;
104
105 if (c < 0x80) {
106 WRITE1((unsigned char)c)
107 NEXT(1, 1)
108 continue;
109 }
110 UCS4INVALID(c)
111
112 REQUIRE_OUTBUF(2)
113
114 GBK_ENCODE(c, code)
115 else return 1;
116
117 OUT1((code >> 8) | 0x80)
118 if (code & 0x8000)
119 OUT2((code & 0xFF)) /* MSB set: GBK */
120 else
121 OUT2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */
122 NEXT(1, 2)
123 }
124
125 return 0;
126}
127
128DECODER(gbk)
129{
130 while (inleft > 0) {
131 unsigned char c = IN1;
132
133 REQUIRE_OUTBUF(1)
134
135 if (c < 0x80) {
136 OUT1(c)
137 NEXT(1, 1)
138 continue;
139 }
140
141 REQUIRE_INBUF(2)
142
143 GBK_DECODE(c, IN2, **outbuf)
144 else return 2;
145
146 NEXT(2, 1)
147 }
148
149 return 0;
150}
151
152
153/*
154 * GB18030 codec
155 */
156
157ENCODER(gb18030)
158{
159 while (inleft > 0) {
160 ucs4_t c = IN1;
161 DBCHAR code;
162
163 if (c < 0x80) {
164 WRITE1(c)
165 NEXT(1, 1)
166 continue;
167 }
168
169 DECODE_SURROGATE(c)
170 if (c > 0x10FFFF)
171#if Py_UNICODE_SIZE == 2
172 return 2; /* surrogates pair */
173#else
174 return 1;
175#endif
176 else if (c >= 0x10000) {
177 ucs4_t tc = c - 0x10000;
178
179 REQUIRE_OUTBUF(4)
180
181 OUT4((unsigned char)(tc % 10) + 0x30)
182 tc /= 10;
183 OUT3((unsigned char)(tc % 126) + 0x81)
184 tc /= 126;
185 OUT2((unsigned char)(tc % 10) + 0x30)
186 tc /= 10;
187 OUT1((unsigned char)(tc + 0x90))
188
189#if Py_UNICODE_SIZE == 2
190 NEXT(2, 4) /* surrogates pair */
191#else
192 NEXT(1, 4)
193#endif
194 continue;
195 }
196
197 REQUIRE_OUTBUF(2)
198
199 GBK_ENCODE(c, code)
200 else TRYMAP_ENC(gb18030ext, code, c);
201 else {
202 const struct _gb18030_to_unibmp_ranges *utrrange;
203
204 REQUIRE_OUTBUF(4)
205
206 for (utrrange = gb18030_to_unibmp_ranges;
207 utrrange->first != 0;
208 utrrange++)
209 if (utrrange->first <= c &&
210 c <= utrrange->last) {
211 Py_UNICODE tc;
212
213 tc = c - utrrange->first +
214 utrrange->base;
215
216 OUT4((unsigned char)(tc % 10) + 0x30)
217 tc /= 10;
218 OUT3((unsigned char)(tc % 126) + 0x81)
219 tc /= 126;
220 OUT2((unsigned char)(tc % 10) + 0x30)
221 tc /= 10;
222 OUT1((unsigned char)tc + 0x81)
223
224 NEXT(1, 4)
225 break;
226 }
227
228 if (utrrange->first == 0)
229 return 1;
230 continue;
231 }
232
233 OUT1((code >> 8) | 0x80)
234 if (code & 0x8000)
235 OUT2((code & 0xFF)) /* MSB set: GBK or GB18030ext */
236 else
237 OUT2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */
238
239 NEXT(1, 2)
240 }
241
242 return 0;
243}
244
245DECODER(gb18030)
246{
247 while (inleft > 0) {
248 unsigned char c = IN1, c2;
249
250 REQUIRE_OUTBUF(1)
251
252 if (c < 0x80) {
253 OUT1(c)
254 NEXT(1, 1)
255 continue;
256 }
257
258 REQUIRE_INBUF(2)
259
260 c2 = IN2;
261 if (c2 >= 0x30 && c2 <= 0x39) { /* 4 bytes seq */
262 const struct _gb18030_to_unibmp_ranges *utr;
263 unsigned char c3, c4;
264 ucs4_t lseq;
265
266 REQUIRE_INBUF(4)
267 c3 = IN3;
268 c4 = IN4;
269 if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39)
270 return 4;
271 c -= 0x81; c2 -= 0x30;
272 c3 -= 0x81; c4 -= 0x30;
273
274 if (c < 4) { /* U+0080 - U+FFFF */
275 lseq = ((ucs4_t)c * 10 + c2) * 1260 +
276 (ucs4_t)c3 * 10 + c4;
277 if (lseq < 39420) {
278 for (utr = gb18030_to_unibmp_ranges;
279 lseq >= (utr + 1)->base;
280 utr++) ;
281 OUT1(utr->first - utr->base + lseq)
282 NEXT(4, 1)
283 continue;
284 }
285 }
286 else if (c >= 15) { /* U+10000 - U+10FFFF */
287 lseq = 0x10000 + (((ucs4_t)c-15) * 10 + c2)
288 * 1260 + (ucs4_t)c3 * 10 + c4;
289 if (lseq <= 0x10FFFF) {
290 WRITEUCS4(lseq);
291 NEXT_IN(4)
292 continue;
293 }
294 }
295 return 4;
296 }
297
298 GBK_DECODE(c, c2, **outbuf)
299 else TRYMAP_DEC(gb18030ext, **outbuf, c, c2);
300 else return 2;
301
302 NEXT(2, 1)
303 }
304
305 return 0;
306}
307
308
309/*
310 * HZ codec
311 */
312
313ENCODER_INIT(hz)
314{
315 state->i = 0;
316 return 0;
317}
318
319ENCODER_RESET(hz)
320{
321 if (state->i != 0) {
322 WRITE2('~', '}')
323 state->i = 0;
324 NEXT_OUT(2)
325 }
326 return 0;
327}
328
329ENCODER(hz)
330{
331 while (inleft > 0) {
332 Py_UNICODE c = IN1;
333 DBCHAR code;
334
335 if (c < 0x80) {
336 if (state->i == 0) {
337 WRITE1((unsigned char)c)
338 NEXT(1, 1)
339 }
340 else {
341 WRITE3('~', '}', (unsigned char)c)
342 NEXT(1, 3)
343 state->i = 0;
344 }
345 continue;
346 }
347
348 UCS4INVALID(c)
349
350 TRYMAP_ENC(gbcommon, code, c);
351 else return 1;
352
353 if (code & 0x8000) /* MSB set: GBK */
354 return 1;
355
356 if (state->i == 0) {
357 WRITE4('~', '{', code >> 8, code & 0xff)
358 NEXT(1, 4)
359 state->i = 1;
360 }
361 else {
362 WRITE2(code >> 8, code & 0xff)
363 NEXT(1, 2)
364 }
365 }
366
367 return 0;
368}
369
370DECODER_INIT(hz)
371{
372 state->i = 0;
373 return 0;
374}
375
376DECODER_RESET(hz)
377{
378 state->i = 0;
379 return 0;
380}
381
382DECODER(hz)
383{
384 while (inleft > 0) {
385 unsigned char c = IN1;
386
387 if (c == '~') {
388 unsigned char c2 = IN2;
389
390 REQUIRE_INBUF(2)
391 if (c2 == '~') {
392 WRITE1('~')
393 NEXT(2, 1)
394 continue;
395 }
396 else if (c2 == '{' && state->i == 0)
397 state->i = 1; /* set GB */
398 else if (c2 == '}' && state->i == 1)
399 state->i = 0; /* set ASCII */
400 else if (c2 == '\n')
401 ; /* line-continuation */
402 else
403 return 2;
404 NEXT(2, 0);
405 continue;
406 }
407
408 if (c & 0x80)
409 return 1;
410
411 if (state->i == 0) { /* ASCII mode */
412 WRITE1(c)
413 NEXT(1, 1)
414 }
415 else { /* GB mode */
416 REQUIRE_INBUF(2)
417 REQUIRE_OUTBUF(1)
418 TRYMAP_DEC(gb2312, **outbuf, c, IN2) {
419 NEXT(2, 1)
420 }
421 else
422 return 2;
423 }
424 }
425
426 return 0;
427}
428
429
430BEGIN_MAPPINGS_LIST
431 MAPPING_DECONLY(gb2312)
432 MAPPING_DECONLY(gbkext)
433 MAPPING_ENCONLY(gbcommon)
434 MAPPING_ENCDEC(gb18030ext)
435END_MAPPINGS_LIST
436
437BEGIN_CODECS_LIST
438 CODEC_STATELESS(gb2312)
439 CODEC_STATELESS(gbk)
440 CODEC_STATELESS(gb18030)
441 CODEC_STATEFUL(hz)
442END_CODECS_LIST
443
444I_AM_A_MODULE_FOR(cn)
Note: See TracBrowser for help on using the repository browser.