source: python/trunk/Modules/cjkcodecs/cjkcodecs.h

Last change on this file was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 13.3 KB
RevLine 
[2]1/*
2 * cjkcodecs.h: common header for cjkcodecs
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7#ifndef _CJKCODECS_H_
8#define _CJKCODECS_H_
9
10#define PY_SSIZE_T_CLEAN
11#include "Python.h"
12#include "multibytecodec.h"
13
14
15/* a unicode "undefined" codepoint */
[391]16#define UNIINV 0xFFFE
[2]17
18/* internal-use DBCS codepoints which aren't used by any charsets */
[391]19#define NOCHAR 0xFFFF
20#define MULTIC 0xFFFE
21#define DBCINV 0xFFFD
[2]22
23/* shorter macros to save source size of mapping tables */
24#define U UNIINV
25#define N NOCHAR
26#define M MULTIC
27#define D DBCINV
28
29struct dbcs_index {
[391]30 const ucs2_t *map;
31 unsigned char bottom, top;
[2]32};
33typedef struct dbcs_index decode_map;
34
35struct widedbcs_index {
[391]36 const ucs4_t *map;
37 unsigned char bottom, top;
[2]38};
39typedef struct widedbcs_index widedecode_map;
40
41struct unim_index {
[391]42 const DBCHAR *map;
43 unsigned char bottom, top;
[2]44};
45typedef struct unim_index encode_map;
46
47struct unim_index_bytebased {
[391]48 const unsigned char *map;
49 unsigned char bottom, top;
[2]50};
51
52struct dbcs_map {
[391]53 const char *charset;
54 const struct unim_index *encmap;
55 const struct dbcs_index *decmap;
[2]56};
57
58struct pair_encodemap {
[391]59 ucs4_t uniseq;
60 DBCHAR code;
[2]61};
62
63static const MultibyteCodec *codec_list;
64static const struct dbcs_map *mapping_list;
65
[391]66#define CODEC_INIT(encoding) \
67 static int encoding##_codec_init(const void *config)
[2]68
[391]69#define ENCODER_INIT(encoding) \
70 static int encoding##_encode_init( \
71 MultibyteCodec_State *state, const void *config)
72#define ENCODER(encoding) \
73 static Py_ssize_t encoding##_encode( \
74 MultibyteCodec_State *state, const void *config, \
75 const Py_UNICODE **inbuf, Py_ssize_t inleft, \
76 unsigned char **outbuf, Py_ssize_t outleft, int flags)
77#define ENCODER_RESET(encoding) \
78 static Py_ssize_t encoding##_encode_reset( \
79 MultibyteCodec_State *state, const void *config, \
80 unsigned char **outbuf, Py_ssize_t outleft)
[2]81
[391]82#define DECODER_INIT(encoding) \
83 static int encoding##_decode_init( \
84 MultibyteCodec_State *state, const void *config)
85#define DECODER(encoding) \
86 static Py_ssize_t encoding##_decode( \
87 MultibyteCodec_State *state, const void *config, \
88 const unsigned char **inbuf, Py_ssize_t inleft, \
89 Py_UNICODE **outbuf, Py_ssize_t outleft)
90#define DECODER_RESET(encoding) \
91 static Py_ssize_t encoding##_decode_reset( \
92 MultibyteCodec_State *state, const void *config)
[2]93
94#if Py_UNICODE_SIZE == 4
[391]95#define UCS4INVALID(code) \
96 if ((code) > 0xFFFF) \
97 return 1;
[2]98#else
[391]99#define UCS4INVALID(code) \
100 if (0) ;
[2]101#endif
102
[391]103#define NEXT_IN(i) \
104 (*inbuf) += (i); \
105 (inleft) -= (i);
106#define NEXT_OUT(o) \
107 (*outbuf) += (o); \
108 (outleft) -= (o);
109#define NEXT(i, o) \
110 NEXT_IN(i) NEXT_OUT(o)
[2]111
[391]112#define REQUIRE_INBUF(n) \
113 if (inleft < (n)) \
114 return MBERR_TOOFEW;
115#define REQUIRE_OUTBUF(n) \
116 if (outleft < (n)) \
117 return MBERR_TOOSMALL;
[2]118
119#define IN1 ((*inbuf)[0])
120#define IN2 ((*inbuf)[1])
121#define IN3 ((*inbuf)[2])
122#define IN4 ((*inbuf)[3])
123
124#define OUT1(c) ((*outbuf)[0]) = (c);
125#define OUT2(c) ((*outbuf)[1]) = (c);
126#define OUT3(c) ((*outbuf)[2]) = (c);
127#define OUT4(c) ((*outbuf)[3]) = (c);
128
[391]129#define WRITE1(c1) \
130 REQUIRE_OUTBUF(1) \
131 (*outbuf)[0] = (c1);
132#define WRITE2(c1, c2) \
133 REQUIRE_OUTBUF(2) \
134 (*outbuf)[0] = (c1); \
135 (*outbuf)[1] = (c2);
136#define WRITE3(c1, c2, c3) \
137 REQUIRE_OUTBUF(3) \
138 (*outbuf)[0] = (c1); \
139 (*outbuf)[1] = (c2); \
140 (*outbuf)[2] = (c3);
141#define WRITE4(c1, c2, c3, c4) \
142 REQUIRE_OUTBUF(4) \
143 (*outbuf)[0] = (c1); \
144 (*outbuf)[1] = (c2); \
145 (*outbuf)[2] = (c3); \
146 (*outbuf)[3] = (c4);
[2]147
148#if Py_UNICODE_SIZE == 2
[391]149# define WRITEUCS4(c) \
150 REQUIRE_OUTBUF(2) \
151 (*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10); \
152 (*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff); \
153 NEXT_OUT(2)
[2]154#else
[391]155# define WRITEUCS4(c) \
156 REQUIRE_OUTBUF(1) \
157 **outbuf = (Py_UNICODE)(c); \
158 NEXT_OUT(1)
[2]159#endif
160
[391]161#define _TRYMAP_ENC(m, assi, val) \
162 ((m)->map != NULL && (val) >= (m)->bottom && \
163 (val)<= (m)->top && ((assi) = (m)->map[(val) - \
164 (m)->bottom]) != NOCHAR)
165#define TRYMAP_ENC_COND(charset, assi, uni) \
166 _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
167#define TRYMAP_ENC(charset, assi, uni) \
168 if TRYMAP_ENC_COND(charset, assi, uni)
[2]169
[391]170#define _TRYMAP_DEC(m, assi, val) \
171 ((m)->map != NULL && (val) >= (m)->bottom && \
172 (val)<= (m)->top && ((assi) = (m)->map[(val) - \
173 (m)->bottom]) != UNIINV)
174#define TRYMAP_DEC(charset, assi, c1, c2) \
175 if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
[2]176
[391]177#define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val) \
178 ((m)->map != NULL && (val) >= (m)->bottom && \
179 (val)<= (m)->top && \
180 ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \
181 (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \
182 (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1))
183#define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \
184 if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
185 assplane, asshi, asslo, (uni) & 0xff)
186#define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2) \
187 if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
[2]188
189#if Py_UNICODE_SIZE == 2
[391]190#define DECODE_SURROGATE(c) \
191 if (c >> 10 == 0xd800 >> 10) { /* high surrogate */ \
192 REQUIRE_INBUF(2) \
193 if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \
194 c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \
195 ((ucs4_t)(IN2) - 0xdc00); \
196 } \
197 }
198#define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1)
[2]199#else
200#define DECODE_SURROGATE(c) {;}
[391]201#define GET_INSIZE(c) 1
[2]202#endif
203
204#define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = {
205#define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
206#define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
207#define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap},
[391]208#define END_MAPPINGS_LIST \
209 {"", NULL, NULL} }; \
210 static const struct dbcs_map *mapping_list = \
211 (const struct dbcs_map *)_mapping_list;
[2]212
213#define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = {
[391]214#define _STATEFUL_METHODS(enc) \
215 enc##_encode, \
216 enc##_encode_init, \
217 enc##_encode_reset, \
218 enc##_decode, \
219 enc##_decode_init, \
220 enc##_decode_reset,
221#define _STATELESS_METHODS(enc) \
222 enc##_encode, NULL, NULL, \
223 enc##_decode, NULL, NULL,
224#define CODEC_STATEFUL(enc) { \
225 #enc, NULL, NULL, \
226 _STATEFUL_METHODS(enc) \
[2]227},
[391]228#define CODEC_STATELESS(enc) { \
229 #enc, NULL, NULL, \
230 _STATELESS_METHODS(enc) \
[2]231},
[391]232#define CODEC_STATELESS_WINIT(enc) { \
233 #enc, NULL, \
234 enc##_codec_init, \
235 _STATELESS_METHODS(enc) \
[2]236},
[391]237#define END_CODECS_LIST \
238 {"", NULL,} }; \
239 static const MultibyteCodec *codec_list = \
240 (const MultibyteCodec *)_codec_list;
[2]241
242static PyObject *
243getmultibytecodec(void)
244{
[391]245 static PyObject *cofunc = NULL;
[2]246
[391]247 if (cofunc == NULL) {
248 PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec");
249 if (mod == NULL)
250 return NULL;
251 cofunc = PyObject_GetAttrString(mod, "__create_codec");
252 Py_DECREF(mod);
253 }
254 return cofunc;
[2]255}
256
257static PyObject *
258getcodec(PyObject *self, PyObject *encoding)
259{
[391]260 PyObject *codecobj, *r, *cofunc;
261 const MultibyteCodec *codec;
262 const char *enc;
[2]263
[391]264 if (!PyString_Check(encoding)) {
265 PyErr_SetString(PyExc_TypeError,
266 "encoding name must be a string.");
267 return NULL;
268 }
[2]269
[391]270 cofunc = getmultibytecodec();
271 if (cofunc == NULL)
272 return NULL;
[2]273
[391]274 enc = PyString_AS_STRING(encoding);
275 for (codec = codec_list; codec->encoding[0]; codec++)
276 if (strcmp(codec->encoding, enc) == 0)
277 break;
[2]278
[391]279 if (codec->encoding[0] == '\0') {
280 PyErr_SetString(PyExc_LookupError,
281 "no such codec is supported.");
282 return NULL;
283 }
[2]284
[391]285 codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL);
286 if (codecobj == NULL)
287 return NULL;
[2]288
[391]289 r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
290 Py_DECREF(codecobj);
[2]291
[391]292 return r;
[2]293}
294
295static struct PyMethodDef __methods[] = {
[391]296 {"getcodec", (PyCFunction)getcodec, METH_O, ""},
297 {NULL, NULL},
[2]298};
299
300static int
301register_maps(PyObject *module)
302{
[391]303 const struct dbcs_map *h;
[2]304
[391]305 for (h = mapping_list; h->charset[0] != '\0'; h++) {
306 char mhname[256] = "__map_";
307 int r;
308 strcpy(mhname + sizeof("__map_") - 1, h->charset);
309 r = PyModule_AddObject(module, mhname,
310 PyCapsule_New((void *)h, PyMultibyteCodec_CAPSULE_NAME, NULL));
311 if (r == -1)
312 return -1;
313 }
314 return 0;
[2]315}
316
317#ifdef USING_BINARY_PAIR_SEARCH
318static DBCHAR
319find_pairencmap(ucs2_t body, ucs2_t modifier,
[391]320 const struct pair_encodemap *haystack, int haystacksize)
[2]321{
[391]322 int pos, min, max;
323 ucs4_t value = body << 16 | modifier;
[2]324
[391]325 min = 0;
326 max = haystacksize;
[2]327
[391]328 for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
329 if (value < haystack[pos].uniseq) {
330 if (max == pos) break;
331 else max = pos;
332 }
333 else if (value > haystack[pos].uniseq) {
334 if (min == pos) break;
335 else min = pos;
336 }
337 else
338 break;
[2]339
[391]340 if (value == haystack[pos].uniseq)
341 return haystack[pos].code;
342 else
343 return DBCINV;
[2]344}
345#endif
346
347#ifdef USING_IMPORTED_MAPS
348#define IMPORT_MAP(locale, charset, encmap, decmap) \
[391]349 importmap("_codecs_" #locale, "__map_" #charset, \
350 (const void**)encmap, (const void**)decmap)
[2]351
352static int
353importmap(const char *modname, const char *symbol,
[391]354 const void **encmap, const void **decmap)
[2]355{
[391]356 PyObject *o, *mod;
[2]357
[391]358 mod = PyImport_ImportModule((char *)modname);
359 if (mod == NULL)
360 return -1;
[2]361
[391]362 o = PyObject_GetAttrString(mod, (char*)symbol);
363 if (o == NULL)
364 goto errorexit;
365 else if (!PyCapsule_IsValid(o, PyMultibyteCodec_CAPSULE_NAME)) {
366 PyErr_SetString(PyExc_ValueError,
367 "map data must be a Capsule.");
368 goto errorexit;
369 }
370 else {
371 struct dbcs_map *map;
372 map = PyCapsule_GetPointer(o, PyMultibyteCodec_CAPSULE_NAME);
373 if (encmap != NULL)
374 *encmap = map->encmap;
375 if (decmap != NULL)
376 *decmap = map->decmap;
377 Py_DECREF(o);
378 }
[2]379
[391]380 Py_DECREF(mod);
381 return 0;
[2]382
383errorexit:
[391]384 Py_DECREF(mod);
385 return -1;
[2]386}
387#endif
388
[10]389#ifdef __OS2__
[391]390#define I_AM_A_MODULE_FOR(loc) \
391 PyMODINIT_FUNC \
392 init_codecs_##loc(void) \
393 { \
394 PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\
395 if (m != NULL) \
396 (void)register_maps(m); \
397 }
[10]398#else
[391]399#define I_AM_A_MODULE_FOR(loc) \
400 void \
401 init_codecs_##loc(void) \
402 { \
403 PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\
404 if (m != NULL) \
405 (void)register_maps(m); \
406 }
[10]407#endif
[2]408
409#endif
Note: See TracBrowser for help on using the repository browser.