Context Navigation

← Previous Revision
Next Revision →
Blame
Revision Log

xmltok_impl.c

Last change on this file was 391, checked in by dmik, 11 years ago
python: Merge vendor 2.7.6 to trunk.
Property svn:eol-style set to `native`
File size: 43.5 KB

Line
1	/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd
2	See the file COPYING for copying permission.
3	*/
4
5	/* This file is included! */
6	#ifdef XML_TOK_IMPL_C
7
8	#ifndef IS_INVALID_CHAR
9	#define IS_INVALID_CHAR(enc, ptr, n) (0)
10	#endif
11
12	#define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \
13	case BT_LEAD ## n: \
14	if (end - ptr < n) \
15	return XML_TOK_PARTIAL_CHAR; \
16	if (IS_INVALID_CHAR(enc, ptr, n)) { \
17	*(nextTokPtr) = (ptr); \
18	return XML_TOK_INVALID; \
19	} \
20	ptr += n; \
21	break;
22
23	#define INVALID_CASES(ptr, nextTokPtr) \
24	INVALID_LEAD_CASE(2, ptr, nextTokPtr) \
25	INVALID_LEAD_CASE(3, ptr, nextTokPtr) \
26	INVALID_LEAD_CASE(4, ptr, nextTokPtr) \
27	case BT_NONXML: \
28	case BT_MALFORM: \
29	case BT_TRAIL: \
30	*(nextTokPtr) = (ptr); \
31	return XML_TOK_INVALID;
32
33	#define CHECK_NAME_CASE(n, enc, ptr, end, nextTokPtr) \
34	case BT_LEAD ## n: \
35	if (end - ptr < n) \
36	return XML_TOK_PARTIAL_CHAR; \
37	if (!IS_NAME_CHAR(enc, ptr, n)) { \
38	*nextTokPtr = ptr; \
39	return XML_TOK_INVALID; \
40	} \
41	ptr += n; \
42	break;
43
44	#define CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) \
45	case BT_NONASCII: \
46	if (!IS_NAME_CHAR_MINBPC(enc, ptr)) { \
47	*nextTokPtr = ptr; \
48	return XML_TOK_INVALID; \
49	} \
50	case BT_NMSTRT: \
51	case BT_HEX: \
52	case BT_DIGIT: \
53	case BT_NAME: \
54	case BT_MINUS: \
55	ptr += MINBPC(enc); \
56	break; \
57	CHECK_NAME_CASE(2, enc, ptr, end, nextTokPtr) \
58	CHECK_NAME_CASE(3, enc, ptr, end, nextTokPtr) \
59	CHECK_NAME_CASE(4, enc, ptr, end, nextTokPtr)
60
61	#define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \
62	case BT_LEAD ## n: \
63	if (end - ptr < n) \
64	return XML_TOK_PARTIAL_CHAR; \
65	if (!IS_NMSTRT_CHAR(enc, ptr, n)) { \
66	*nextTokPtr = ptr; \
67	return XML_TOK_INVALID; \
68	} \
69	ptr += n; \
70	break;
71
72	#define CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) \
73	case BT_NONASCII: \
74	if (!IS_NMSTRT_CHAR_MINBPC(enc, ptr)) { \
75	*nextTokPtr = ptr; \
76	return XML_TOK_INVALID; \
77	} \
78	case BT_NMSTRT: \
79	case BT_HEX: \
80	ptr += MINBPC(enc); \
81	break; \
82	CHECK_NMSTRT_CASE(2, enc, ptr, end, nextTokPtr) \
83	CHECK_NMSTRT_CASE(3, enc, ptr, end, nextTokPtr) \
84	CHECK_NMSTRT_CASE(4, enc, ptr, end, nextTokPtr)
85
86	#ifndef PREFIX
87	#define PREFIX(ident) ident
88	#endif
89
90	/* ptr points to character following "<!-" */
91
92	static int PTRCALL
93	PREFIX(scanComment)(const ENCODING enc, const char ptr,
94	const char end, const char *nextTokPtr)
95	{
96	if (ptr != end) {
97	if (!CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
98	*nextTokPtr = ptr;
99	return XML_TOK_INVALID;
100	}
101	ptr += MINBPC(enc);
102	while (ptr != end) {
103	switch (BYTE_TYPE(enc, ptr)) {
104	INVALID_CASES(ptr, nextTokPtr)
105	case BT_MINUS:
106	if ((ptr += MINBPC(enc)) == end)
107	return XML_TOK_PARTIAL;
108	if (CHAR_MATCHES(enc, ptr, ASCII_MINUS)) {
109	if ((ptr += MINBPC(enc)) == end)
110	return XML_TOK_PARTIAL;
111	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
112	*nextTokPtr = ptr;
113	return XML_TOK_INVALID;
114	}
115	*nextTokPtr = ptr + MINBPC(enc);
116	return XML_TOK_COMMENT;
117	}
118	break;
119	default:
120	ptr += MINBPC(enc);
121	break;
122	}
123	}
124	}
125	return XML_TOK_PARTIAL;
126	}
127
128	/* ptr points to character following "<!" */
129
130	static int PTRCALL
131	PREFIX(scanDecl)(const ENCODING enc, const char ptr,
132	const char end, const char *nextTokPtr)
133	{
134	if (ptr == end)
135	return XML_TOK_PARTIAL;
136	switch (BYTE_TYPE(enc, ptr)) {
137	case BT_MINUS:
138	return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
139	case BT_LSQB:
140	*nextTokPtr = ptr + MINBPC(enc);
141	return XML_TOK_COND_SECT_OPEN;
142	case BT_NMSTRT:
143	case BT_HEX:
144	ptr += MINBPC(enc);
145	break;
146	default:
147	*nextTokPtr = ptr;
148	return XML_TOK_INVALID;
149	}
150	while (ptr != end) {
151	switch (BYTE_TYPE(enc, ptr)) {
152	case BT_PERCNT:
153	if (ptr + MINBPC(enc) == end)
154	return XML_TOK_PARTIAL;
155	/* don't allow <!ENTITY% foo "whatever"> */
156	switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) {
157	case BT_S: case BT_CR: case BT_LF: case BT_PERCNT:
158	*nextTokPtr = ptr;
159	return XML_TOK_INVALID;
160	}
161	/* fall through */
162	case BT_S: case BT_CR: case BT_LF:
163	*nextTokPtr = ptr;
164	return XML_TOK_DECL_OPEN;
165	case BT_NMSTRT:
166	case BT_HEX:
167	ptr += MINBPC(enc);
168	break;
169	default:
170	*nextTokPtr = ptr;
171	return XML_TOK_INVALID;
172	}
173	}
174	return XML_TOK_PARTIAL;
175	}
176
177	static int PTRCALL
178	PREFIX(checkPiTarget)(const ENCODING enc, const char ptr,
179	const char end, int tokPtr)
180	{
181	int upper = 0;
182	*tokPtr = XML_TOK_PI;
183	if (end - ptr != MINBPC(enc)*3)
184	return 1;
185	switch (BYTE_TO_ASCII(enc, ptr)) {
186	case ASCII_x:
187	break;
188	case ASCII_X:
189	upper = 1;
190	break;
191	default:
192	return 1;
193	}
194	ptr += MINBPC(enc);
195	switch (BYTE_TO_ASCII(enc, ptr)) {
196	case ASCII_m:
197	break;
198	case ASCII_M:
199	upper = 1;
200	break;
201	default:
202	return 1;
203	}
204	ptr += MINBPC(enc);
205	switch (BYTE_TO_ASCII(enc, ptr)) {
206	case ASCII_l:
207	break;
208	case ASCII_L:
209	upper = 1;
210	break;
211	default:
212	return 1;
213	}
214	if (upper)
215	return 0;
216	*tokPtr = XML_TOK_XML_DECL;
217	return 1;
218	}
219
220	/* ptr points to character following "<?" */
221
222	static int PTRCALL
223	PREFIX(scanPi)(const ENCODING enc, const char ptr,
224	const char end, const char *nextTokPtr)
225	{
226	int tok;
227	const char *target = ptr;
228	if (ptr == end)
229	return XML_TOK_PARTIAL;
230	switch (BYTE_TYPE(enc, ptr)) {
231	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
232	default:
233	*nextTokPtr = ptr;
234	return XML_TOK_INVALID;
235	}
236	while (ptr != end) {
237	switch (BYTE_TYPE(enc, ptr)) {
238	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
239	case BT_S: case BT_CR: case BT_LF:
240	if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
241	*nextTokPtr = ptr;
242	return XML_TOK_INVALID;
243	}
244	ptr += MINBPC(enc);
245	while (ptr != end) {
246	switch (BYTE_TYPE(enc, ptr)) {
247	INVALID_CASES(ptr, nextTokPtr)
248	case BT_QUEST:
249	ptr += MINBPC(enc);
250	if (ptr == end)
251	return XML_TOK_PARTIAL;
252	if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {
253	*nextTokPtr = ptr + MINBPC(enc);
254	return tok;
255	}
256	break;
257	default:
258	ptr += MINBPC(enc);
259	break;
260	}
261	}
262	return XML_TOK_PARTIAL;
263	case BT_QUEST:
264	if (!PREFIX(checkPiTarget)(enc, target, ptr, &tok)) {
265	*nextTokPtr = ptr;
266	return XML_TOK_INVALID;
267	}
268	ptr += MINBPC(enc);
269	if (ptr == end)
270	return XML_TOK_PARTIAL;
271	if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {
272	*nextTokPtr = ptr + MINBPC(enc);
273	return tok;
274	}
275	/* fall through */
276	default:
277	*nextTokPtr = ptr;
278	return XML_TOK_INVALID;
279	}
280	}
281	return XML_TOK_PARTIAL;
282	}
283
284	static int PTRCALL
285	PREFIX(scanCdataSection)(const ENCODING enc, const char ptr,
286	const char end, const char *nextTokPtr)
287	{
288	static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A,
289	ASCII_T, ASCII_A, ASCII_LSQB };
290	int i;
291	/* CDATA[ */
292	if (end - ptr < 6 * MINBPC(enc))
293	return XML_TOK_PARTIAL;
294	for (i = 0; i < 6; i++, ptr += MINBPC(enc)) {
295	if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) {
296	*nextTokPtr = ptr;
297	return XML_TOK_INVALID;
298	}
299	}
300	*nextTokPtr = ptr;
301	return XML_TOK_CDATA_SECT_OPEN;
302	}
303
304	static int PTRCALL
305	PREFIX(cdataSectionTok)(const ENCODING enc, const char ptr,
306	const char end, const char *nextTokPtr)
307	{
308	if (ptr == end)
309	return XML_TOK_NONE;
310	if (MINBPC(enc) > 1) {
311	size_t n = end - ptr;
312	if (n & (MINBPC(enc) - 1)) {
313	n &= ~(MINBPC(enc) - 1);
314	if (n == 0)
315	return XML_TOK_PARTIAL;
316	end = ptr + n;
317	}
318	}
319	switch (BYTE_TYPE(enc, ptr)) {
320	case BT_RSQB:
321	ptr += MINBPC(enc);
322	if (ptr == end)
323	return XML_TOK_PARTIAL;
324	if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
325	break;
326	ptr += MINBPC(enc);
327	if (ptr == end)
328	return XML_TOK_PARTIAL;
329	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
330	ptr -= MINBPC(enc);
331	break;
332	}
333	*nextTokPtr = ptr + MINBPC(enc);
334	return XML_TOK_CDATA_SECT_CLOSE;
335	case BT_CR:
336	ptr += MINBPC(enc);
337	if (ptr == end)
338	return XML_TOK_PARTIAL;
339	if (BYTE_TYPE(enc, ptr) == BT_LF)
340	ptr += MINBPC(enc);
341	*nextTokPtr = ptr;
342	return XML_TOK_DATA_NEWLINE;
343	case BT_LF:
344	*nextTokPtr = ptr + MINBPC(enc);
345	return XML_TOK_DATA_NEWLINE;
346	INVALID_CASES(ptr, nextTokPtr)
347	default:
348	ptr += MINBPC(enc);
349	break;
350	}
351	while (ptr != end) {
352	switch (BYTE_TYPE(enc, ptr)) {
353	#define LEAD_CASE(n) \
354	case BT_LEAD ## n: \
355	if (end - ptr < n \|\| IS_INVALID_CHAR(enc, ptr, n)) { \
356	*nextTokPtr = ptr; \
357	return XML_TOK_DATA_CHARS; \
358	} \
359	ptr += n; \
360	break;
361	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
362	#undef LEAD_CASE
363	case BT_NONXML:
364	case BT_MALFORM:
365	case BT_TRAIL:
366	case BT_CR:
367	case BT_LF:
368	case BT_RSQB:
369	*nextTokPtr = ptr;
370	return XML_TOK_DATA_CHARS;
371	default:
372	ptr += MINBPC(enc);
373	break;
374	}
375	}
376	*nextTokPtr = ptr;
377	return XML_TOK_DATA_CHARS;
378	}
379
380	/* ptr points to character following "</" */
381
382	static int PTRCALL
383	PREFIX(scanEndTag)(const ENCODING enc, const char ptr,
384	const char end, const char *nextTokPtr)
385	{
386	if (ptr == end)
387	return XML_TOK_PARTIAL;
388	switch (BYTE_TYPE(enc, ptr)) {
389	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
390	default:
391	*nextTokPtr = ptr;
392	return XML_TOK_INVALID;
393	}
394	while (ptr != end) {
395	switch (BYTE_TYPE(enc, ptr)) {
396	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
397	case BT_S: case BT_CR: case BT_LF:
398	for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {
399	switch (BYTE_TYPE(enc, ptr)) {
400	case BT_S: case BT_CR: case BT_LF:
401	break;
402	case BT_GT:
403	*nextTokPtr = ptr + MINBPC(enc);
404	return XML_TOK_END_TAG;
405	default:
406	*nextTokPtr = ptr;
407	return XML_TOK_INVALID;
408	}
409	}
410	return XML_TOK_PARTIAL;
411	#ifdef XML_NS
412	case BT_COLON:
413	/* no need to check qname syntax here,
414	since end-tag must match exactly */
415	ptr += MINBPC(enc);
416	break;
417	#endif
418	case BT_GT:
419	*nextTokPtr = ptr + MINBPC(enc);
420	return XML_TOK_END_TAG;
421	default:
422	*nextTokPtr = ptr;
423	return XML_TOK_INVALID;
424	}
425	}
426	return XML_TOK_PARTIAL;
427	}
428
429	/* ptr points to character following "&#X" */
430
431	static int PTRCALL
432	PREFIX(scanHexCharRef)(const ENCODING enc, const char ptr,
433	const char end, const char *nextTokPtr)
434	{
435	if (ptr != end) {
436	switch (BYTE_TYPE(enc, ptr)) {
437	case BT_DIGIT:
438	case BT_HEX:
439	break;
440	default:
441	*nextTokPtr = ptr;
442	return XML_TOK_INVALID;
443	}
444	for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {
445	switch (BYTE_TYPE(enc, ptr)) {
446	case BT_DIGIT:
447	case BT_HEX:
448	break;
449	case BT_SEMI:
450	*nextTokPtr = ptr + MINBPC(enc);
451	return XML_TOK_CHAR_REF;
452	default:
453	*nextTokPtr = ptr;
454	return XML_TOK_INVALID;
455	}
456	}
457	}
458	return XML_TOK_PARTIAL;
459	}
460
461	/* ptr points to character following "&#" */
462
463	static int PTRCALL
464	PREFIX(scanCharRef)(const ENCODING enc, const char ptr,
465	const char end, const char *nextTokPtr)
466	{
467	if (ptr != end) {
468	if (CHAR_MATCHES(enc, ptr, ASCII_x))
469	return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
470	switch (BYTE_TYPE(enc, ptr)) {
471	case BT_DIGIT:
472	break;
473	default:
474	*nextTokPtr = ptr;
475	return XML_TOK_INVALID;
476	}
477	for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) {
478	switch (BYTE_TYPE(enc, ptr)) {
479	case BT_DIGIT:
480	break;
481	case BT_SEMI:
482	*nextTokPtr = ptr + MINBPC(enc);
483	return XML_TOK_CHAR_REF;
484	default:
485	*nextTokPtr = ptr;
486	return XML_TOK_INVALID;
487	}
488	}
489	}
490	return XML_TOK_PARTIAL;
491	}
492
493	/* ptr points to character following "&" */
494
495	static int PTRCALL
496	PREFIX(scanRef)(const ENCODING enc, const char ptr, const char *end,
497	const char **nextTokPtr)
498	{
499	if (ptr == end)
500	return XML_TOK_PARTIAL;
501	switch (BYTE_TYPE(enc, ptr)) {
502	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
503	case BT_NUM:
504	return PREFIX(scanCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
505	default:
506	*nextTokPtr = ptr;
507	return XML_TOK_INVALID;
508	}
509	while (ptr != end) {
510	switch (BYTE_TYPE(enc, ptr)) {
511	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
512	case BT_SEMI:
513	*nextTokPtr = ptr + MINBPC(enc);
514	return XML_TOK_ENTITY_REF;
515	default:
516	*nextTokPtr = ptr;
517	return XML_TOK_INVALID;
518	}
519	}
520	return XML_TOK_PARTIAL;
521	}
522
523	/* ptr points to character following first character of attribute name */
524
525	static int PTRCALL
526	PREFIX(scanAtts)(const ENCODING enc, const char ptr, const char *end,
527	const char **nextTokPtr)
528	{
529	#ifdef XML_NS
530	int hadColon = 0;
531	#endif
532	while (ptr != end) {
533	switch (BYTE_TYPE(enc, ptr)) {
534	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
535	#ifdef XML_NS
536	case BT_COLON:
537	if (hadColon) {
538	*nextTokPtr = ptr;
539	return XML_TOK_INVALID;
540	}
541	hadColon = 1;
542	ptr += MINBPC(enc);
543	if (ptr == end)
544	return XML_TOK_PARTIAL;
545	switch (BYTE_TYPE(enc, ptr)) {
546	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
547	default:
548	*nextTokPtr = ptr;
549	return XML_TOK_INVALID;
550	}
551	break;
552	#endif
553	case BT_S: case BT_CR: case BT_LF:
554	for (;;) {
555	int t;
556
557	ptr += MINBPC(enc);
558	if (ptr == end)
559	return XML_TOK_PARTIAL;
560	t = BYTE_TYPE(enc, ptr);
561	if (t == BT_EQUALS)
562	break;
563	switch (t) {
564	case BT_S:
565	case BT_LF:
566	case BT_CR:
567	break;
568	default:
569	*nextTokPtr = ptr;
570	return XML_TOK_INVALID;
571	}
572	}
573	/* fall through */
574	case BT_EQUALS:
575	{
576	int open;
577	#ifdef XML_NS
578	hadColon = 0;
579	#endif
580	for (;;) {
581	ptr += MINBPC(enc);
582	if (ptr == end)
583	return XML_TOK_PARTIAL;
584	open = BYTE_TYPE(enc, ptr);
585	if (open == BT_QUOT \|\| open == BT_APOS)
586	break;
587	switch (open) {
588	case BT_S:
589	case BT_LF:
590	case BT_CR:
591	break;
592	default:
593	*nextTokPtr = ptr;
594	return XML_TOK_INVALID;
595	}
596	}
597	ptr += MINBPC(enc);
598	/* in attribute value */
599	for (;;) {
600	int t;
601	if (ptr == end)
602	return XML_TOK_PARTIAL;
603	t = BYTE_TYPE(enc, ptr);
604	if (t == open)
605	break;
606	switch (t) {
607	INVALID_CASES(ptr, nextTokPtr)
608	case BT_AMP:
609	{
610	int tok = PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, &ptr);
611	if (tok <= 0) {
612	if (tok == XML_TOK_INVALID)
613	*nextTokPtr = ptr;
614	return tok;
615	}
616	break;
617	}
618	case BT_LT:
619	*nextTokPtr = ptr;
620	return XML_TOK_INVALID;
621	default:
622	ptr += MINBPC(enc);
623	break;
624	}
625	}
626	ptr += MINBPC(enc);
627	if (ptr == end)
628	return XML_TOK_PARTIAL;
629	switch (BYTE_TYPE(enc, ptr)) {
630	case BT_S:
631	case BT_CR:
632	case BT_LF:
633	break;
634	case BT_SOL:
635	goto sol;
636	case BT_GT:
637	goto gt;
638	default:
639	*nextTokPtr = ptr;
640	return XML_TOK_INVALID;
641	}
642	/* ptr points to closing quote */
643	for (;;) {
644	ptr += MINBPC(enc);
645	if (ptr == end)
646	return XML_TOK_PARTIAL;
647	switch (BYTE_TYPE(enc, ptr)) {
648	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
649	case BT_S: case BT_CR: case BT_LF:
650	continue;
651	case BT_GT:
652	gt:
653	*nextTokPtr = ptr + MINBPC(enc);
654	return XML_TOK_START_TAG_WITH_ATTS;
655	case BT_SOL:
656	sol:
657	ptr += MINBPC(enc);
658	if (ptr == end)
659	return XML_TOK_PARTIAL;
660	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
661	*nextTokPtr = ptr;
662	return XML_TOK_INVALID;
663	}
664	*nextTokPtr = ptr + MINBPC(enc);
665	return XML_TOK_EMPTY_ELEMENT_WITH_ATTS;
666	default:
667	*nextTokPtr = ptr;
668	return XML_TOK_INVALID;
669	}
670	break;
671	}
672	break;
673	}
674	default:
675	*nextTokPtr = ptr;
676	return XML_TOK_INVALID;
677	}
678	}
679	return XML_TOK_PARTIAL;
680	}
681
682	/* ptr points to character following "<" */
683
684	static int PTRCALL
685	PREFIX(scanLt)(const ENCODING enc, const char ptr, const char *end,
686	const char **nextTokPtr)
687	{
688	#ifdef XML_NS
689	int hadColon;
690	#endif
691	if (ptr == end)
692	return XML_TOK_PARTIAL;
693	switch (BYTE_TYPE(enc, ptr)) {
694	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
695	case BT_EXCL:
696	if ((ptr += MINBPC(enc)) == end)
697	return XML_TOK_PARTIAL;
698	switch (BYTE_TYPE(enc, ptr)) {
699	case BT_MINUS:
700	return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr);
701	case BT_LSQB:
702	return PREFIX(scanCdataSection)(enc, ptr + MINBPC(enc),
703	end, nextTokPtr);
704	}
705	*nextTokPtr = ptr;
706	return XML_TOK_INVALID;
707	case BT_QUEST:
708	return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
709	case BT_SOL:
710	return PREFIX(scanEndTag)(enc, ptr + MINBPC(enc), end, nextTokPtr);
711	default:
712	*nextTokPtr = ptr;
713	return XML_TOK_INVALID;
714	}
715	#ifdef XML_NS
716	hadColon = 0;
717	#endif
718	/* we have a start-tag */
719	while (ptr != end) {
720	switch (BYTE_TYPE(enc, ptr)) {
721	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
722	#ifdef XML_NS
723	case BT_COLON:
724	if (hadColon) {
725	*nextTokPtr = ptr;
726	return XML_TOK_INVALID;
727	}
728	hadColon = 1;
729	ptr += MINBPC(enc);
730	if (ptr == end)
731	return XML_TOK_PARTIAL;
732	switch (BYTE_TYPE(enc, ptr)) {
733	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
734	default:
735	*nextTokPtr = ptr;
736	return XML_TOK_INVALID;
737	}
738	break;
739	#endif
740	case BT_S: case BT_CR: case BT_LF:
741	{
742	ptr += MINBPC(enc);
743	while (ptr != end) {
744	switch (BYTE_TYPE(enc, ptr)) {
745	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
746	case BT_GT:
747	goto gt;
748	case BT_SOL:
749	goto sol;
750	case BT_S: case BT_CR: case BT_LF:
751	ptr += MINBPC(enc);
752	continue;
753	default:
754	*nextTokPtr = ptr;
755	return XML_TOK_INVALID;
756	}
757	return PREFIX(scanAtts)(enc, ptr, end, nextTokPtr);
758	}
759	return XML_TOK_PARTIAL;
760	}
761	case BT_GT:
762	gt:
763	*nextTokPtr = ptr + MINBPC(enc);
764	return XML_TOK_START_TAG_NO_ATTS;
765	case BT_SOL:
766	sol:
767	ptr += MINBPC(enc);
768	if (ptr == end)
769	return XML_TOK_PARTIAL;
770	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
771	*nextTokPtr = ptr;
772	return XML_TOK_INVALID;
773	}
774	*nextTokPtr = ptr + MINBPC(enc);
775	return XML_TOK_EMPTY_ELEMENT_NO_ATTS;
776	default:
777	*nextTokPtr = ptr;
778	return XML_TOK_INVALID;
779	}
780	}
781	return XML_TOK_PARTIAL;
782	}
783
784	static int PTRCALL
785	PREFIX(contentTok)(const ENCODING enc, const char ptr, const char *end,
786	const char **nextTokPtr)
787	{
788	if (ptr == end)
789	return XML_TOK_NONE;
790	if (MINBPC(enc) > 1) {
791	size_t n = end - ptr;
792	if (n & (MINBPC(enc) - 1)) {
793	n &= ~(MINBPC(enc) - 1);
794	if (n == 0)
795	return XML_TOK_PARTIAL;
796	end = ptr + n;
797	}
798	}
799	switch (BYTE_TYPE(enc, ptr)) {
800	case BT_LT:
801	return PREFIX(scanLt)(enc, ptr + MINBPC(enc), end, nextTokPtr);
802	case BT_AMP:
803	return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
804	case BT_CR:
805	ptr += MINBPC(enc);
806	if (ptr == end)
807	return XML_TOK_TRAILING_CR;
808	if (BYTE_TYPE(enc, ptr) == BT_LF)
809	ptr += MINBPC(enc);
810	*nextTokPtr = ptr;
811	return XML_TOK_DATA_NEWLINE;
812	case BT_LF:
813	*nextTokPtr = ptr + MINBPC(enc);
814	return XML_TOK_DATA_NEWLINE;
815	case BT_RSQB:
816	ptr += MINBPC(enc);
817	if (ptr == end)
818	return XML_TOK_TRAILING_RSQB;
819	if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB))
820	break;
821	ptr += MINBPC(enc);
822	if (ptr == end)
823	return XML_TOK_TRAILING_RSQB;
824	if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) {
825	ptr -= MINBPC(enc);
826	break;
827	}
828	*nextTokPtr = ptr;
829	return XML_TOK_INVALID;
830	INVALID_CASES(ptr, nextTokPtr)
831	default:
832	ptr += MINBPC(enc);
833	break;
834	}
835	while (ptr != end) {
836	switch (BYTE_TYPE(enc, ptr)) {
837	#define LEAD_CASE(n) \
838	case BT_LEAD ## n: \
839	if (end - ptr < n \|\| IS_INVALID_CHAR(enc, ptr, n)) { \
840	*nextTokPtr = ptr; \
841	return XML_TOK_DATA_CHARS; \
842	} \
843	ptr += n; \
844	break;
845	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
846	#undef LEAD_CASE
847	case BT_RSQB:
848	if (ptr + MINBPC(enc) != end) {
849	if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) {
850	ptr += MINBPC(enc);
851	break;
852	}
853	if (ptr + 2*MINBPC(enc) != end) {
854	if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) {
855	ptr += MINBPC(enc);
856	break;
857	}
858	nextTokPtr = ptr + 2MINBPC(enc);
859	return XML_TOK_INVALID;
860	}
861	}
862	/* fall through */
863	case BT_AMP:
864	case BT_LT:
865	case BT_NONXML:
866	case BT_MALFORM:
867	case BT_TRAIL:
868	case BT_CR:
869	case BT_LF:
870	*nextTokPtr = ptr;
871	return XML_TOK_DATA_CHARS;
872	default:
873	ptr += MINBPC(enc);
874	break;
875	}
876	}
877	*nextTokPtr = ptr;
878	return XML_TOK_DATA_CHARS;
879	}
880
881	/* ptr points to character following "%" */
882
883	static int PTRCALL
884	PREFIX(scanPercent)(const ENCODING enc, const char ptr, const char *end,
885	const char **nextTokPtr)
886	{
887	if (ptr == end)
888	return XML_TOK_PARTIAL;
889	switch (BYTE_TYPE(enc, ptr)) {
890	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
891	case BT_S: case BT_LF: case BT_CR: case BT_PERCNT:
892	*nextTokPtr = ptr;
893	return XML_TOK_PERCENT;
894	default:
895	*nextTokPtr = ptr;
896	return XML_TOK_INVALID;
897	}
898	while (ptr != end) {
899	switch (BYTE_TYPE(enc, ptr)) {
900	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
901	case BT_SEMI:
902	*nextTokPtr = ptr + MINBPC(enc);
903	return XML_TOK_PARAM_ENTITY_REF;
904	default:
905	*nextTokPtr = ptr;
906	return XML_TOK_INVALID;
907	}
908	}
909	return XML_TOK_PARTIAL;
910	}
911
912	static int PTRCALL
913	PREFIX(scanPoundName)(const ENCODING enc, const char ptr, const char *end,
914	const char **nextTokPtr)
915	{
916	if (ptr == end)
917	return XML_TOK_PARTIAL;
918	switch (BYTE_TYPE(enc, ptr)) {
919	CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr)
920	default:
921	*nextTokPtr = ptr;
922	return XML_TOK_INVALID;
923	}
924	while (ptr != end) {
925	switch (BYTE_TYPE(enc, ptr)) {
926	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
927	case BT_CR: case BT_LF: case BT_S:
928	case BT_RPAR: case BT_GT: case BT_PERCNT: case BT_VERBAR:
929	*nextTokPtr = ptr;
930	return XML_TOK_POUND_NAME;
931	default:
932	*nextTokPtr = ptr;
933	return XML_TOK_INVALID;
934	}
935	}
936	return -XML_TOK_POUND_NAME;
937	}
938
939	static int PTRCALL
940	PREFIX(scanLit)(int open, const ENCODING *enc,
941	const char ptr, const char end,
942	const char **nextTokPtr)
943	{
944	while (ptr != end) {
945	int t = BYTE_TYPE(enc, ptr);
946	switch (t) {
947	INVALID_CASES(ptr, nextTokPtr)
948	case BT_QUOT:
949	case BT_APOS:
950	ptr += MINBPC(enc);
951	if (t != open)
952	break;
953	if (ptr == end)
954	return -XML_TOK_LITERAL;
955	*nextTokPtr = ptr;
956	switch (BYTE_TYPE(enc, ptr)) {
957	case BT_S: case BT_CR: case BT_LF:
958	case BT_GT: case BT_PERCNT: case BT_LSQB:
959	return XML_TOK_LITERAL;
960	default:
961	return XML_TOK_INVALID;
962	}
963	default:
964	ptr += MINBPC(enc);
965	break;
966	}
967	}
968	return XML_TOK_PARTIAL;
969	}
970
971	static int PTRCALL
972	PREFIX(prologTok)(const ENCODING enc, const char ptr, const char *end,
973	const char **nextTokPtr)
974	{
975	int tok;
976	if (ptr == end)
977	return XML_TOK_NONE;
978	if (MINBPC(enc) > 1) {
979	size_t n = end - ptr;
980	if (n & (MINBPC(enc) - 1)) {
981	n &= ~(MINBPC(enc) - 1);
982	if (n == 0)
983	return XML_TOK_PARTIAL;
984	end = ptr + n;
985	}
986	}
987	switch (BYTE_TYPE(enc, ptr)) {
988	case BT_QUOT:
989	return PREFIX(scanLit)(BT_QUOT, enc, ptr + MINBPC(enc), end, nextTokPtr);
990	case BT_APOS:
991	return PREFIX(scanLit)(BT_APOS, enc, ptr + MINBPC(enc), end, nextTokPtr);
992	case BT_LT:
993	{
994	ptr += MINBPC(enc);
995	if (ptr == end)
996	return XML_TOK_PARTIAL;
997	switch (BYTE_TYPE(enc, ptr)) {
998	case BT_EXCL:
999	return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1000	case BT_QUEST:
1001	return PREFIX(scanPi)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1002	case BT_NMSTRT:
1003	case BT_HEX:
1004	case BT_NONASCII:
1005	case BT_LEAD2:
1006	case BT_LEAD3:
1007	case BT_LEAD4:
1008	*nextTokPtr = ptr - MINBPC(enc);
1009	return XML_TOK_INSTANCE_START;
1010	}
1011	*nextTokPtr = ptr;
1012	return XML_TOK_INVALID;
1013	}
1014	case BT_CR:
1015	if (ptr + MINBPC(enc) == end) {
1016	*nextTokPtr = end;
1017	/* indicate that this might be part of a CR/LF pair */
1018	return -XML_TOK_PROLOG_S;
1019	}
1020	/* fall through */
1021	case BT_S: case BT_LF:
1022	for (;;) {
1023	ptr += MINBPC(enc);
1024	if (ptr == end)
1025	break;
1026	switch (BYTE_TYPE(enc, ptr)) {
1027	case BT_S: case BT_LF:
1028	break;
1029	case BT_CR:
1030	/* don't split CR/LF pair */
1031	if (ptr + MINBPC(enc) != end)
1032	break;
1033	/* fall through */
1034	default:
1035	*nextTokPtr = ptr;
1036	return XML_TOK_PROLOG_S;
1037	}
1038	}
1039	*nextTokPtr = ptr;
1040	return XML_TOK_PROLOG_S;
1041	case BT_PERCNT:
1042	return PREFIX(scanPercent)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1043	case BT_COMMA:
1044	*nextTokPtr = ptr + MINBPC(enc);
1045	return XML_TOK_COMMA;
1046	case BT_LSQB:
1047	*nextTokPtr = ptr + MINBPC(enc);
1048	return XML_TOK_OPEN_BRACKET;
1049	case BT_RSQB:
1050	ptr += MINBPC(enc);
1051	if (ptr == end)
1052	return -XML_TOK_CLOSE_BRACKET;
1053	if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
1054	if (ptr + MINBPC(enc) == end)
1055	return XML_TOK_PARTIAL;
1056	if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) {
1057	nextTokPtr = ptr + 2MINBPC(enc);
1058	return XML_TOK_COND_SECT_CLOSE;
1059	}
1060	}
1061	*nextTokPtr = ptr;
1062	return XML_TOK_CLOSE_BRACKET;
1063	case BT_LPAR:
1064	*nextTokPtr = ptr + MINBPC(enc);
1065	return XML_TOK_OPEN_PAREN;
1066	case BT_RPAR:
1067	ptr += MINBPC(enc);
1068	if (ptr == end)
1069	return -XML_TOK_CLOSE_PAREN;
1070	switch (BYTE_TYPE(enc, ptr)) {
1071	case BT_AST:
1072	*nextTokPtr = ptr + MINBPC(enc);
1073	return XML_TOK_CLOSE_PAREN_ASTERISK;
1074	case BT_QUEST:
1075	*nextTokPtr = ptr + MINBPC(enc);
1076	return XML_TOK_CLOSE_PAREN_QUESTION;
1077	case BT_PLUS:
1078	*nextTokPtr = ptr + MINBPC(enc);
1079	return XML_TOK_CLOSE_PAREN_PLUS;
1080	case BT_CR: case BT_LF: case BT_S:
1081	case BT_GT: case BT_COMMA: case BT_VERBAR:
1082	case BT_RPAR:
1083	*nextTokPtr = ptr;
1084	return XML_TOK_CLOSE_PAREN;
1085	}
1086	*nextTokPtr = ptr;
1087	return XML_TOK_INVALID;
1088	case BT_VERBAR:
1089	*nextTokPtr = ptr + MINBPC(enc);
1090	return XML_TOK_OR;
1091	case BT_GT:
1092	*nextTokPtr = ptr + MINBPC(enc);
1093	return XML_TOK_DECL_CLOSE;
1094	case BT_NUM:
1095	return PREFIX(scanPoundName)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1096	#define LEAD_CASE(n) \
1097	case BT_LEAD ## n: \
1098	if (end - ptr < n) \
1099	return XML_TOK_PARTIAL_CHAR; \
1100	if (IS_NMSTRT_CHAR(enc, ptr, n)) { \
1101	ptr += n; \
1102	tok = XML_TOK_NAME; \
1103	break; \
1104	} \
1105	if (IS_NAME_CHAR(enc, ptr, n)) { \
1106	ptr += n; \
1107	tok = XML_TOK_NMTOKEN; \
1108	break; \
1109	} \
1110	*nextTokPtr = ptr; \
1111	return XML_TOK_INVALID;
1112	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1113	#undef LEAD_CASE
1114	case BT_NMSTRT:
1115	case BT_HEX:
1116	tok = XML_TOK_NAME;
1117	ptr += MINBPC(enc);
1118	break;
1119	case BT_DIGIT:
1120	case BT_NAME:
1121	case BT_MINUS:
1122	#ifdef XML_NS
1123	case BT_COLON:
1124	#endif
1125	tok = XML_TOK_NMTOKEN;
1126	ptr += MINBPC(enc);
1127	break;
1128	case BT_NONASCII:
1129	if (IS_NMSTRT_CHAR_MINBPC(enc, ptr)) {
1130	ptr += MINBPC(enc);
1131	tok = XML_TOK_NAME;
1132	break;
1133	}
1134	if (IS_NAME_CHAR_MINBPC(enc, ptr)) {
1135	ptr += MINBPC(enc);
1136	tok = XML_TOK_NMTOKEN;
1137	break;
1138	}
1139	/* fall through */
1140	default:
1141	*nextTokPtr = ptr;
1142	return XML_TOK_INVALID;
1143	}
1144	while (ptr != end) {
1145	switch (BYTE_TYPE(enc, ptr)) {
1146	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
1147	case BT_GT: case BT_RPAR: case BT_COMMA:
1148	case BT_VERBAR: case BT_LSQB: case BT_PERCNT:
1149	case BT_S: case BT_CR: case BT_LF:
1150	*nextTokPtr = ptr;
1151	return tok;
1152	#ifdef XML_NS
1153	case BT_COLON:
1154	ptr += MINBPC(enc);
1155	switch (tok) {
1156	case XML_TOK_NAME:
1157	if (ptr == end)
1158	return XML_TOK_PARTIAL;
1159	tok = XML_TOK_PREFIXED_NAME;
1160	switch (BYTE_TYPE(enc, ptr)) {
1161	CHECK_NAME_CASES(enc, ptr, end, nextTokPtr)
1162	default:
1163	tok = XML_TOK_NMTOKEN;
1164	break;
1165	}
1166	break;
1167	case XML_TOK_PREFIXED_NAME:
1168	tok = XML_TOK_NMTOKEN;
1169	break;
1170	}
1171	break;
1172	#endif
1173	case BT_PLUS:
1174	if (tok == XML_TOK_NMTOKEN) {
1175	*nextTokPtr = ptr;
1176	return XML_TOK_INVALID;
1177	}
1178	*nextTokPtr = ptr + MINBPC(enc);
1179	return XML_TOK_NAME_PLUS;
1180	case BT_AST:
1181	if (tok == XML_TOK_NMTOKEN) {
1182	*nextTokPtr = ptr;
1183	return XML_TOK_INVALID;
1184	}
1185	*nextTokPtr = ptr + MINBPC(enc);
1186	return XML_TOK_NAME_ASTERISK;
1187	case BT_QUEST:
1188	if (tok == XML_TOK_NMTOKEN) {
1189	*nextTokPtr = ptr;
1190	return XML_TOK_INVALID;
1191	}
1192	*nextTokPtr = ptr + MINBPC(enc);
1193	return XML_TOK_NAME_QUESTION;
1194	default:
1195	*nextTokPtr = ptr;
1196	return XML_TOK_INVALID;
1197	}
1198	}
1199	return -tok;
1200	}
1201
1202	static int PTRCALL
1203	PREFIX(attributeValueTok)(const ENCODING enc, const char ptr,
1204	const char end, const char *nextTokPtr)
1205	{
1206	const char *start;
1207	if (ptr == end)
1208	return XML_TOK_NONE;
1209	start = ptr;
1210	while (ptr != end) {
1211	switch (BYTE_TYPE(enc, ptr)) {
1212	#define LEAD_CASE(n) \
1213	case BT_LEAD ## n: ptr += n; break;
1214	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1215	#undef LEAD_CASE
1216	case BT_AMP:
1217	if (ptr == start)
1218	return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1219	*nextTokPtr = ptr;
1220	return XML_TOK_DATA_CHARS;
1221	case BT_LT:
1222	/* this is for inside entity references */
1223	*nextTokPtr = ptr;
1224	return XML_TOK_INVALID;
1225	case BT_LF:
1226	if (ptr == start) {
1227	*nextTokPtr = ptr + MINBPC(enc);
1228	return XML_TOK_DATA_NEWLINE;
1229	}
1230	*nextTokPtr = ptr;
1231	return XML_TOK_DATA_CHARS;
1232	case BT_CR:
1233	if (ptr == start) {
1234	ptr += MINBPC(enc);
1235	if (ptr == end)
1236	return XML_TOK_TRAILING_CR;
1237	if (BYTE_TYPE(enc, ptr) == BT_LF)
1238	ptr += MINBPC(enc);
1239	*nextTokPtr = ptr;
1240	return XML_TOK_DATA_NEWLINE;
1241	}
1242	*nextTokPtr = ptr;
1243	return XML_TOK_DATA_CHARS;
1244	case BT_S:
1245	if (ptr == start) {
1246	*nextTokPtr = ptr + MINBPC(enc);
1247	return XML_TOK_ATTRIBUTE_VALUE_S;
1248	}
1249	*nextTokPtr = ptr;
1250	return XML_TOK_DATA_CHARS;
1251	default:
1252	ptr += MINBPC(enc);
1253	break;
1254	}
1255	}
1256	*nextTokPtr = ptr;
1257	return XML_TOK_DATA_CHARS;
1258	}
1259
1260	static int PTRCALL
1261	PREFIX(entityValueTok)(const ENCODING enc, const char ptr,
1262	const char end, const char *nextTokPtr)
1263	{
1264	const char *start;
1265	if (ptr == end)
1266	return XML_TOK_NONE;
1267	start = ptr;
1268	while (ptr != end) {
1269	switch (BYTE_TYPE(enc, ptr)) {
1270	#define LEAD_CASE(n) \
1271	case BT_LEAD ## n: ptr += n; break;
1272	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1273	#undef LEAD_CASE
1274	case BT_AMP:
1275	if (ptr == start)
1276	return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr);
1277	*nextTokPtr = ptr;
1278	return XML_TOK_DATA_CHARS;
1279	case BT_PERCNT:
1280	if (ptr == start) {
1281	int tok = PREFIX(scanPercent)(enc, ptr + MINBPC(enc),
1282	end, nextTokPtr);
1283	return (tok == XML_TOK_PERCENT) ? XML_TOK_INVALID : tok;
1284	}
1285	*nextTokPtr = ptr;
1286	return XML_TOK_DATA_CHARS;
1287	case BT_LF:
1288	if (ptr == start) {
1289	*nextTokPtr = ptr + MINBPC(enc);
1290	return XML_TOK_DATA_NEWLINE;
1291	}
1292	*nextTokPtr = ptr;
1293	return XML_TOK_DATA_CHARS;
1294	case BT_CR:
1295	if (ptr == start) {
1296	ptr += MINBPC(enc);
1297	if (ptr == end)
1298	return XML_TOK_TRAILING_CR;
1299	if (BYTE_TYPE(enc, ptr) == BT_LF)
1300	ptr += MINBPC(enc);
1301	*nextTokPtr = ptr;
1302	return XML_TOK_DATA_NEWLINE;
1303	}
1304	*nextTokPtr = ptr;
1305	return XML_TOK_DATA_CHARS;
1306	default:
1307	ptr += MINBPC(enc);
1308	break;
1309	}
1310	}
1311	*nextTokPtr = ptr;
1312	return XML_TOK_DATA_CHARS;
1313	}
1314
1315	#ifdef XML_DTD
1316
1317	static int PTRCALL
1318	PREFIX(ignoreSectionTok)(const ENCODING enc, const char ptr,
1319	const char end, const char *nextTokPtr)
1320	{
1321	int level = 0;
1322	if (MINBPC(enc) > 1) {
1323	size_t n = end - ptr;
1324	if (n & (MINBPC(enc) - 1)) {
1325	n &= ~(MINBPC(enc) - 1);
1326	end = ptr + n;
1327	}
1328	}
1329	while (ptr != end) {
1330	switch (BYTE_TYPE(enc, ptr)) {
1331	INVALID_CASES(ptr, nextTokPtr)
1332	case BT_LT:
1333	if ((ptr += MINBPC(enc)) == end)
1334	return XML_TOK_PARTIAL;
1335	if (CHAR_MATCHES(enc, ptr, ASCII_EXCL)) {
1336	if ((ptr += MINBPC(enc)) == end)
1337	return XML_TOK_PARTIAL;
1338	if (CHAR_MATCHES(enc, ptr, ASCII_LSQB)) {
1339	++level;
1340	ptr += MINBPC(enc);
1341	}
1342	}
1343	break;
1344	case BT_RSQB:
1345	if ((ptr += MINBPC(enc)) == end)
1346	return XML_TOK_PARTIAL;
1347	if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) {
1348	if ((ptr += MINBPC(enc)) == end)
1349	return XML_TOK_PARTIAL;
1350	if (CHAR_MATCHES(enc, ptr, ASCII_GT)) {
1351	ptr += MINBPC(enc);
1352	if (level == 0) {
1353	*nextTokPtr = ptr;
1354	return XML_TOK_IGNORE_SECT;
1355	}
1356	--level;
1357	}
1358	}
1359	break;
1360	default:
1361	ptr += MINBPC(enc);
1362	break;
1363	}
1364	}
1365	return XML_TOK_PARTIAL;
1366	}
1367
1368	#endif /* XML_DTD */
1369
1370	static int PTRCALL
1371	PREFIX(isPublicId)(const ENCODING enc, const char ptr, const char *end,
1372	const char **badPtr)
1373	{
1374	ptr += MINBPC(enc);
1375	end -= MINBPC(enc);
1376	for (; ptr != end; ptr += MINBPC(enc)) {
1377	switch (BYTE_TYPE(enc, ptr)) {
1378	case BT_DIGIT:
1379	case BT_HEX:
1380	case BT_MINUS:
1381	case BT_APOS:
1382	case BT_LPAR:
1383	case BT_RPAR:
1384	case BT_PLUS:
1385	case BT_COMMA:
1386	case BT_SOL:
1387	case BT_EQUALS:
1388	case BT_QUEST:
1389	case BT_CR:
1390	case BT_LF:
1391	case BT_SEMI:
1392	case BT_EXCL:
1393	case BT_AST:
1394	case BT_PERCNT:
1395	case BT_NUM:
1396	#ifdef XML_NS
1397	case BT_COLON:
1398	#endif
1399	break;
1400	case BT_S:
1401	if (CHAR_MATCHES(enc, ptr, ASCII_TAB)) {
1402	*badPtr = ptr;
1403	return 0;
1404	}
1405	break;
1406	case BT_NAME:
1407	case BT_NMSTRT:
1408	if (!(BYTE_TO_ASCII(enc, ptr) & ~0x7f))
1409	break;
1410	default:
1411	switch (BYTE_TO_ASCII(enc, ptr)) {
1412	case 0x24: /* $ */
1413	case 0x40: /* @ */
1414	break;
1415	default:
1416	*badPtr = ptr;
1417	return 0;
1418	}
1419	break;
1420	}
1421	}
1422	return 1;
1423	}
1424
1425	/* This must only be called for a well-formed start-tag or empty
1426	element tag. Returns the number of attributes. Pointers to the
1427	first attsMax attributes are stored in atts.
1428	*/
1429
1430	static int PTRCALL
1431	PREFIX(getAtts)(const ENCODING enc, const char ptr,
1432	int attsMax, ATTRIBUTE *atts)
1433	{
1434	enum { other, inName, inValue } state = inName;
1435	int nAtts = 0;
1436	int open = 0; /* defined when state == inValue;
1437	initialization just to shut up compilers */
1438
1439	for (ptr += MINBPC(enc);; ptr += MINBPC(enc)) {
1440	switch (BYTE_TYPE(enc, ptr)) {
1441	#define START_NAME \
1442	if (state == other) { \
1443	if (nAtts < attsMax) { \
1444	atts[nAtts].name = ptr; \
1445	atts[nAtts].normalized = 1; \
1446	} \
1447	state = inName; \
1448	}
1449	#define LEAD_CASE(n) \
1450	case BT_LEAD ## n: START_NAME ptr += (n - MINBPC(enc)); break;
1451	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1452	#undef LEAD_CASE
1453	case BT_NONASCII:
1454	case BT_NMSTRT:
1455	case BT_HEX:
1456	START_NAME
1457	break;
1458	#undef START_NAME
1459	case BT_QUOT:
1460	if (state != inValue) {
1461	if (nAtts < attsMax)
1462	atts[nAtts].valuePtr = ptr + MINBPC(enc);
1463	state = inValue;
1464	open = BT_QUOT;
1465	}
1466	else if (open == BT_QUOT) {
1467	state = other;
1468	if (nAtts < attsMax)
1469	atts[nAtts].valueEnd = ptr;
1470	nAtts++;
1471	}
1472	break;
1473	case BT_APOS:
1474	if (state != inValue) {
1475	if (nAtts < attsMax)
1476	atts[nAtts].valuePtr = ptr + MINBPC(enc);
1477	state = inValue;
1478	open = BT_APOS;
1479	}
1480	else if (open == BT_APOS) {
1481	state = other;
1482	if (nAtts < attsMax)
1483	atts[nAtts].valueEnd = ptr;
1484	nAtts++;
1485	}
1486	break;
1487	case BT_AMP:
1488	if (nAtts < attsMax)
1489	atts[nAtts].normalized = 0;
1490	break;
1491	case BT_S:
1492	if (state == inName)
1493	state = other;
1494	else if (state == inValue
1495	&& nAtts < attsMax
1496	&& atts[nAtts].normalized
1497	&& (ptr == atts[nAtts].valuePtr
1498	\|\| BYTE_TO_ASCII(enc, ptr) != ASCII_SPACE
1499	\|\| BYTE_TO_ASCII(enc, ptr + MINBPC(enc)) == ASCII_SPACE
1500	\|\| BYTE_TYPE(enc, ptr + MINBPC(enc)) == open))
1501	atts[nAtts].normalized = 0;
1502	break;
1503	case BT_CR: case BT_LF:
1504	/* This case ensures that the first attribute name is counted
1505	Apart from that we could just change state on the quote. */
1506	if (state == inName)
1507	state = other;
1508	else if (state == inValue && nAtts < attsMax)
1509	atts[nAtts].normalized = 0;
1510	break;
1511	case BT_GT:
1512	case BT_SOL:
1513	if (state != inValue)
1514	return nAtts;
1515	break;
1516	default:
1517	break;
1518	}
1519	}
1520	/* not reached */
1521	}
1522
1523	static int PTRFASTCALL
1524	PREFIX(charRefNumber)(const ENCODING enc, const char ptr)
1525	{
1526	int result = 0;
1527	/* skip &# */
1528	ptr += 2*MINBPC(enc);
1529	if (CHAR_MATCHES(enc, ptr, ASCII_x)) {
1530	for (ptr += MINBPC(enc);
1531	!CHAR_MATCHES(enc, ptr, ASCII_SEMI);
1532	ptr += MINBPC(enc)) {
1533	int c = BYTE_TO_ASCII(enc, ptr);
1534	switch (c) {
1535	case ASCII_0: case ASCII_1: case ASCII_2: case ASCII_3: case ASCII_4:
1536	case ASCII_5: case ASCII_6: case ASCII_7: case ASCII_8: case ASCII_9:
1537	result <<= 4;
1538	result \|= (c - ASCII_0);
1539	break;
1540	case ASCII_A: case ASCII_B: case ASCII_C:
1541	case ASCII_D: case ASCII_E: case ASCII_F:
1542	result <<= 4;
1543	result += 10 + (c - ASCII_A);
1544	break;
1545	case ASCII_a: case ASCII_b: case ASCII_c:
1546	case ASCII_d: case ASCII_e: case ASCII_f:
1547	result <<= 4;
1548	result += 10 + (c - ASCII_a);
1549	break;
1550	}
1551	if (result >= 0x110000)
1552	return -1;
1553	}
1554	}
1555	else {
1556	for (; !CHAR_MATCHES(enc, ptr, ASCII_SEMI); ptr += MINBPC(enc)) {
1557	int c = BYTE_TO_ASCII(enc, ptr);
1558	result *= 10;
1559	result += (c - ASCII_0);
1560	if (result >= 0x110000)
1561	return -1;
1562	}
1563	}
1564	return checkCharRefNumber(result);
1565	}
1566
1567	static int PTRCALL
1568	PREFIX(predefinedEntityName)(const ENCODING enc, const char ptr,
1569	const char *end)
1570	{
1571	switch ((end - ptr)/MINBPC(enc)) {
1572	case 2:
1573	if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_t)) {
1574	switch (BYTE_TO_ASCII(enc, ptr)) {
1575	case ASCII_l:
1576	return ASCII_LT;
1577	case ASCII_g:
1578	return ASCII_GT;
1579	}
1580	}
1581	break;
1582	case 3:
1583	if (CHAR_MATCHES(enc, ptr, ASCII_a)) {
1584	ptr += MINBPC(enc);
1585	if (CHAR_MATCHES(enc, ptr, ASCII_m)) {
1586	ptr += MINBPC(enc);
1587	if (CHAR_MATCHES(enc, ptr, ASCII_p))
1588	return ASCII_AMP;
1589	}
1590	}
1591	break;
1592	case 4:
1593	switch (BYTE_TO_ASCII(enc, ptr)) {
1594	case ASCII_q:
1595	ptr += MINBPC(enc);
1596	if (CHAR_MATCHES(enc, ptr, ASCII_u)) {
1597	ptr += MINBPC(enc);
1598	if (CHAR_MATCHES(enc, ptr, ASCII_o)) {
1599	ptr += MINBPC(enc);
1600	if (CHAR_MATCHES(enc, ptr, ASCII_t))
1601	return ASCII_QUOT;
1602	}
1603	}
1604	break;
1605	case ASCII_a:
1606	ptr += MINBPC(enc);
1607	if (CHAR_MATCHES(enc, ptr, ASCII_p)) {
1608	ptr += MINBPC(enc);
1609	if (CHAR_MATCHES(enc, ptr, ASCII_o)) {
1610	ptr += MINBPC(enc);
1611	if (CHAR_MATCHES(enc, ptr, ASCII_s))
1612	return ASCII_APOS;
1613	}
1614	}
1615	break;
1616	}
1617	}
1618	return 0;
1619	}
1620
1621	static int PTRCALL
1622	PREFIX(sameName)(const ENCODING enc, const char ptr1, const char *ptr2)
1623	{
1624	for (;;) {
1625	switch (BYTE_TYPE(enc, ptr1)) {
1626	#define LEAD_CASE(n) \
1627	case BT_LEAD ## n: \
1628	if (ptr1++ != ptr2++) \
1629	return 0;
1630	LEAD_CASE(4) LEAD_CASE(3) LEAD_CASE(2)
1631	#undef LEAD_CASE
1632	/* fall through */
1633	if (ptr1++ != ptr2++)
1634	return 0;
1635	break;
1636	case BT_NONASCII:
1637	case BT_NMSTRT:
1638	#ifdef XML_NS
1639	case BT_COLON:
1640	#endif
1641	case BT_HEX:
1642	case BT_DIGIT:
1643	case BT_NAME:
1644	case BT_MINUS:
1645	if (ptr2++ != ptr1++)
1646	return 0;
1647	if (MINBPC(enc) > 1) {
1648	if (ptr2++ != ptr1++)
1649	return 0;
1650	if (MINBPC(enc) > 2) {
1651	if (ptr2++ != ptr1++)
1652	return 0;
1653	if (MINBPC(enc) > 3) {
1654	if (ptr2++ != ptr1++)
1655	return 0;
1656	}
1657	}
1658	}
1659	break;
1660	default:
1661	if (MINBPC(enc) == 1 && ptr1 == ptr2)
1662	return 1;
1663	switch (BYTE_TYPE(enc, ptr2)) {
1664	case BT_LEAD2:
1665	case BT_LEAD3:
1666	case BT_LEAD4:
1667	case BT_NONASCII:
1668	case BT_NMSTRT:
1669	#ifdef XML_NS
1670	case BT_COLON:
1671	#endif
1672	case BT_HEX:
1673	case BT_DIGIT:
1674	case BT_NAME:
1675	case BT_MINUS:
1676	return 0;
1677	default:
1678	return 1;
1679	}
1680	}
1681	}
1682	/* not reached */
1683	}
1684
1685	static int PTRCALL
1686	PREFIX(nameMatchesAscii)(const ENCODING enc, const char ptr1,
1687	const char end1, const char ptr2)
1688	{
1689	for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
1690	if (ptr1 == end1)
1691	return 0;
1692	if (!CHAR_MATCHES(enc, ptr1, *ptr2))
1693	return 0;
1694	}
1695	return ptr1 == end1;
1696	}
1697
1698	static int PTRFASTCALL
1699	PREFIX(nameLength)(const ENCODING enc, const char ptr)
1700	{
1701	const char *start = ptr;
1702	for (;;) {
1703	switch (BYTE_TYPE(enc, ptr)) {
1704	#define LEAD_CASE(n) \
1705	case BT_LEAD ## n: ptr += n; break;
1706	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1707	#undef LEAD_CASE
1708	case BT_NONASCII:
1709	case BT_NMSTRT:
1710	#ifdef XML_NS
1711	case BT_COLON:
1712	#endif
1713	case BT_HEX:
1714	case BT_DIGIT:
1715	case BT_NAME:
1716	case BT_MINUS:
1717	ptr += MINBPC(enc);
1718	break;
1719	default:
1720	return (int)(ptr - start);
1721	}
1722	}
1723	}
1724
1725	static const char * PTRFASTCALL
1726	PREFIX(skipS)(const ENCODING enc, const char ptr)
1727	{
1728	for (;;) {
1729	switch (BYTE_TYPE(enc, ptr)) {
1730	case BT_LF:
1731	case BT_CR:
1732	case BT_S:
1733	ptr += MINBPC(enc);
1734	break;
1735	default:
1736	return ptr;
1737	}
1738	}
1739	}
1740
1741	static void PTRCALL
1742	PREFIX(updatePosition)(const ENCODING *enc,
1743	const char *ptr,
1744	const char *end,
1745	POSITION *pos)
1746	{
1747	while (ptr < end) {
1748	switch (BYTE_TYPE(enc, ptr)) {
1749	#define LEAD_CASE(n) \
1750	case BT_LEAD ## n: \
1751	ptr += n; \
1752	break;
1753	LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4)
1754	#undef LEAD_CASE
1755	case BT_LF:
1756	pos->columnNumber = (XML_Size)-1;
1757	pos->lineNumber++;
1758	ptr += MINBPC(enc);
1759	break;
1760	case BT_CR:
1761	pos->lineNumber++;
1762	ptr += MINBPC(enc);
1763	if (ptr != end && BYTE_TYPE(enc, ptr) == BT_LF)
1764	ptr += MINBPC(enc);
1765	pos->columnNumber = (XML_Size)-1;
1766	break;
1767	default:
1768	ptr += MINBPC(enc);
1769	break;
1770	}
1771	pos->columnNumber++;
1772	}
1773	}
1774
1775	#undef DO_LEAD_CASE
1776	#undef MULTIBYTE_CASES
1777	#undef INVALID_CASES
1778	#undef CHECK_NAME_CASE
1779	#undef CHECK_NAME_CASES
1780	#undef CHECK_NMSTRT_CASE
1781	#undef CHECK_NMSTRT_CASES
1782
1783	#endif /* XML_TOK_IMPL_C */

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: python/trunk/Modules/expat/xmltok_impl.c

Download in other formats: