/[dtapublic]/projs/trunk/shared_source/tcl_base/tclutf.c
ViewVC logotype

Contents of /projs/trunk/shared_source/tcl_base/tclutf.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 42 - (show annotations) (download)
Fri Oct 14 01:50:00 2016 UTC (8 years, 2 months ago) by dashley
File MIME type: text/plain
File size: 39815 byte(s)
Move shared source code to commonize.
1 /* $Header: /cvsroot/esrg/sfesrg/esrgpcpj/shared/tcl_base/tclutf.c,v 1.1.1.1 2001/06/13 04:47:01 dtashley Exp $ */
2
3 /*
4 * tclUtf.c --
5 *
6 * Routines for manipulating UTF-8 strings.
7 *
8 * Copyright (c) 1997-1998 Sun Microsystems, Inc.
9 *
10 * See the file "license.terms" for information on usage and redistribution
11 * of this file, and for a DISCLAIMER OF ALL WARRANTIES.
12 *
13 * RCS: @(#) $Id: tclutf.c,v 1.1.1.1 2001/06/13 04:47:01 dtashley Exp $
14 */
15
16 #include "tclInt.h"
17
18 /*
19 * Include the static character classification tables and macros.
20 */
21
22 #include "tclUniData.c"
23
24 /*
25 * The following macros are used for fast character category tests. The
26 * x_BITS values are shifted right by the category value to determine whether
27 * the given category is included in the set.
28 */
29
30 #define ALPHA_BITS ((1 << UPPERCASE_LETTER) | (1 << LOWERCASE_LETTER) \
31 | (1 << TITLECASE_LETTER) | (1 << MODIFIER_LETTER) | (1 << OTHER_LETTER))
32
33 #define DIGIT_BITS (1 << DECIMAL_DIGIT_NUMBER)
34
35 #define SPACE_BITS ((1 << SPACE_SEPARATOR) | (1 << LINE_SEPARATOR) \
36 | (1 << PARAGRAPH_SEPARATOR))
37
38 #define CONNECTOR_BITS (1 << CONNECTOR_PUNCTUATION)
39
40 #define PRINT_BITS (ALPHA_BITS | DIGIT_BITS | SPACE_BITS | \
41 (1 << NON_SPACING_MARK) | (1 << ENCLOSING_MARK) | \
42 (1 << COMBINING_SPACING_MARK) | (1 << LETTER_NUMBER) | \
43 (1 << OTHER_NUMBER) | (1 << CONNECTOR_PUNCTUATION) | \
44 (1 << DASH_PUNCTUATION) | (1 << OPEN_PUNCTUATION) | \
45 (1 << CLOSE_PUNCTUATION) | (1 << INITIAL_QUOTE_PUNCTUATION) | \
46 (1 << FINAL_QUOTE_PUNCTUATION) | (1 << OTHER_PUNCTUATION) | \
47 (1 << MATH_SYMBOL) | (1 << CURRENCY_SYMBOL) | \
48 (1 << MODIFIER_SYMBOL) | (1 << OTHER_SYMBOL))
49
50 #define PUNCT_BITS ((1 << CONNECTOR_PUNCTUATION) | \
51 (1 << DASH_PUNCTUATION) | (1 << OPEN_PUNCTUATION) | \
52 (1 << CLOSE_PUNCTUATION) | (1 << INITIAL_QUOTE_PUNCTUATION) | \
53 (1 << FINAL_QUOTE_PUNCTUATION) | (1 << OTHER_PUNCTUATION))
54
55 /*
56 * Unicode characters less than this value are represented by themselves
57 * in UTF-8 strings.
58 */
59
60 #define UNICODE_SELF 0x80
61
62 /*
63 * The following structures are used when mapping between Unicode (UCS-2)
64 * and UTF-8.
65 */
66
67 CONST unsigned char totalBytes[256] = {
68 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
69 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
70 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
71 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
72 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
73 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
74 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
75 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
76 #if TCL_UTF_MAX > 3
77 4,4,4,4,4,4,4,4,
78 #else
79 1,1,1,1,1,1,1,1,
80 #endif
81 #if TCL_UTF_MAX > 4
82 5,5,5,5,
83 #else
84 1,1,1,1,
85 #endif
86 #if TCL_UTF_MAX > 5
87 6,6,6,6
88 #else
89 1,1,1,1
90 #endif
91 };
92
93 /*
94 * Procedures used only in this module.
95 */
96
97 static int UtfCount _ANSI_ARGS_((int ch));
98
99
100 /*
101 *---------------------------------------------------------------------------
102 *
103 * UtfCount --
104 *
105 * Find the number of bytes in the Utf character "ch".
106 *
107 * Results:
108 * The return values is the number of bytes in the Utf character "ch".
109 *
110 * Side effects:
111 * None.
112 *
113 *---------------------------------------------------------------------------
114 */
115
116 static int
117 UtfCount(ch)
118 int ch; /* The Tcl_UniChar whose size is returned. */
119 {
120 if ((ch > 0) && (ch < UNICODE_SELF)) {
121 return 1;
122 }
123 if (ch <= 0x7FF) {
124 return 2;
125 }
126 if (ch <= 0xFFFF) {
127 return 3;
128 }
129 #if TCL_UTF_MAX > 3
130 if (ch <= 0x1FFFFF) {
131 return 4;
132 }
133 if (ch <= 0x3FFFFFF) {
134 return 5;
135 }
136 if (ch <= 0x7FFFFFFF) {
137 return 6;
138 }
139 #endif
140 return 3;
141 }
142
143 /*
144 *---------------------------------------------------------------------------
145 *
146 * Tcl_UniCharToUtf --
147 *
148 * Store the given Tcl_UniChar as a sequence of UTF-8 bytes in the
149 * provided buffer. Equivalent to Plan 9 runetochar().
150 *
151 * Results:
152 * The return values is the number of bytes in the buffer that
153 * were consumed.
154 *
155 * Side effects:
156 * None.
157 *
158 *---------------------------------------------------------------------------
159 */
160
161 INLINE int
162 Tcl_UniCharToUtf(ch, str)
163 int ch; /* The Tcl_UniChar to be stored in the
164 * buffer. */
165 char *str; /* Buffer in which the UTF-8 representation
166 * of the Tcl_UniChar is stored. Buffer must
167 * be large enough to hold the UTF-8 character
168 * (at most TCL_UTF_MAX bytes). */
169 {
170 if ((ch > 0) && (ch < UNICODE_SELF)) {
171 str[0] = (char) ch;
172 return 1;
173 }
174 if (ch <= 0x7FF) {
175 str[1] = (char) ((ch | 0x80) & 0xBF);
176 str[0] = (char) ((ch >> 6) | 0xC0);
177 return 2;
178 }
179 if (ch <= 0xFFFF) {
180 three:
181 str[2] = (char) ((ch | 0x80) & 0xBF);
182 str[1] = (char) (((ch >> 6) | 0x80) & 0xBF);
183 str[0] = (char) ((ch >> 12) | 0xE0);
184 return 3;
185 }
186
187 #if TCL_UTF_MAX > 3
188 if (ch <= 0x1FFFFF) {
189 str[3] = (char) ((ch | 0x80) & 0xBF);
190 str[2] = (char) (((ch >> 6) | 0x80) & 0xBF);
191 str[1] = (char) (((ch >> 12) | 0x80) & 0xBF);
192 str[0] = (char) ((ch >> 18) | 0xF0);
193 return 4;
194 }
195 if (ch <= 0x3FFFFFF) {
196 str[4] = (char) ((ch | 0x80) & 0xBF);
197 str[3] = (char) (((ch >> 6) | 0x80) & 0xBF);
198 str[2] = (char) (((ch >> 12) | 0x80) & 0xBF);
199 str[1] = (char) (((ch >> 18) | 0x80) & 0xBF);
200 str[0] = (char) ((ch >> 24) | 0xF8);
201 return 5;
202 }
203 if (ch <= 0x7FFFFFFF) {
204 str[5] = (char) ((ch | 0x80) & 0xBF);
205 str[4] = (char) (((ch >> 6) | 0x80) & 0xBF);
206 str[3] = (char) (((ch >> 12) | 0x80) & 0xBF);
207 str[2] = (char) (((ch >> 18) | 0x80) & 0xBF);
208 str[1] = (char) (((ch >> 24) | 0x80) & 0xBF);
209 str[0] = (char) ((ch >> 30) | 0xFC);
210 return 6;
211 }
212 #endif
213
214 ch = 0xFFFD;
215 goto three;
216 }
217
218 /*
219 *---------------------------------------------------------------------------
220 *
221 * Tcl_UniCharToUtfDString --
222 *
223 * Convert the given Unicode string to UTF-8.
224 *
225 * Results:
226 * The return value is a pointer to the UTF-8 representation of the
227 * Unicode string. Storage for the return value is appended to the
228 * end of dsPtr.
229 *
230 * Side effects:
231 * None.
232 *
233 *---------------------------------------------------------------------------
234 */
235
236 char *
237 Tcl_UniCharToUtfDString(wString, numChars, dsPtr)
238 CONST Tcl_UniChar *wString; /* Unicode string to convert to UTF-8. */
239 int numChars; /* Length of Unicode string in Tcl_UniChars
240 * (must be >= 0). */
241 Tcl_DString *dsPtr; /* UTF-8 representation of string is
242 * appended to this previously initialized
243 * DString. */
244 {
245 CONST Tcl_UniChar *w, *wEnd;
246 char *p, *string;
247 int oldLength;
248
249 /*
250 * UTF-8 string length in bytes will be <= Unicode string length *
251 * TCL_UTF_MAX.
252 */
253
254 oldLength = Tcl_DStringLength(dsPtr);
255 Tcl_DStringSetLength(dsPtr, (oldLength + numChars + 1) * TCL_UTF_MAX);
256 string = Tcl_DStringValue(dsPtr) + oldLength;
257
258 p = string;
259 wEnd = wString + numChars;
260 for (w = wString; w < wEnd; ) {
261 p += Tcl_UniCharToUtf(*w, p);
262 w++;
263 }
264 Tcl_DStringSetLength(dsPtr, oldLength + (p - string));
265
266 return string;
267 }
268
269 /*
270 *---------------------------------------------------------------------------
271 *
272 * Tcl_UtfToUniChar --
273 *
274 * Extract the Tcl_UniChar represented by the UTF-8 string. Bad
275 * UTF-8 sequences are converted to valid Tcl_UniChars and processing
276 * continues. Equivalent to Plan 9 chartorune().
277 *
278 * The caller must ensure that the source buffer is long enough that
279 * this routine does not run off the end and dereference non-existent
280 * memory looking for trail bytes. If the source buffer is known to
281 * be '\0' terminated, this cannot happen. Otherwise, the caller
282 * should call Tcl_UtfCharComplete() before calling this routine to
283 * ensure that enough bytes remain in the string.
284 *
285 * Results:
286 * *chPtr is filled with the Tcl_UniChar, and the return value is the
287 * number of bytes from the UTF-8 string that were consumed.
288 *
289 * Side effects:
290 * None.
291 *
292 *---------------------------------------------------------------------------
293 */
294
295 int
296 Tcl_UtfToUniChar(str, chPtr)
297 register CONST char *str; /* The UTF-8 string. */
298 register Tcl_UniChar *chPtr; /* Filled with the Tcl_UniChar represented
299 * by the UTF-8 string. */
300 {
301 register int byte;
302
303 /*
304 * Unroll 1 to 3 byte UTF-8 sequences, use loop to handle longer ones.
305 */
306
307 byte = *((unsigned char *) str);
308 if (byte < 0xC0) {
309 /*
310 * Handles properly formed UTF-8 characters between 0x01 and 0x7F.
311 * Also treats \0 and naked trail bytes 0x80 to 0xBF as valid
312 * characters representing themselves.
313 */
314
315 *chPtr = (Tcl_UniChar) byte;
316 return 1;
317 } else if (byte < 0xE0) {
318 if ((str[1] & 0xC0) == 0x80) {
319 /*
320 * Two-byte-character lead-byte followed by a trail-byte.
321 */
322
323 *chPtr = (Tcl_UniChar) (((byte & 0x1F) << 6) | (str[1] & 0x3F));
324 return 2;
325 }
326 /*
327 * A two-byte-character lead-byte not followed by trail-byte
328 * represents itself.
329 */
330
331 *chPtr = (Tcl_UniChar) byte;
332 return 1;
333 } else if (byte < 0xF0) {
334 if (((str[1] & 0xC0) == 0x80) && ((str[2] & 0xC0) == 0x80)) {
335 /*
336 * Three-byte-character lead byte followed by two trail bytes.
337 */
338
339 *chPtr = (Tcl_UniChar) (((byte & 0x0F) << 12)
340 | ((str[1] & 0x3F) << 6) | (str[2] & 0x3F));
341 return 3;
342 }
343 /*
344 * A three-byte-character lead-byte not followed by two trail-bytes
345 * represents itself.
346 */
347
348 *chPtr = (Tcl_UniChar) byte;
349 return 1;
350 }
351 #if TCL_UTF_MAX > 3
352 else {
353 int ch, total, trail;
354
355 total = totalBytes[byte];
356 trail = total - 1;
357 if (trail > 0) {
358 ch = byte & (0x3F >> trail);
359 do {
360 str++;
361 if ((*str & 0xC0) != 0x80) {
362 *chPtr = byte;
363 return 1;
364 }
365 ch <<= 6;
366 ch |= (*str & 0x3F);
367 trail--;
368 } while (trail > 0);
369 *chPtr = ch;
370 return total;
371 }
372 }
373 #endif
374
375 *chPtr = (Tcl_UniChar) byte;
376 return 1;
377 }
378
379 /*
380 *---------------------------------------------------------------------------
381 *
382 * Tcl_UtfToUniCharDString --
383 *
384 * Convert the UTF-8 string to Unicode.
385 *
386 * Results:
387 * The return value is a pointer to the Unicode representation of the
388 * UTF-8 string. Storage for the return value is appended to the
389 * end of dsPtr. The Unicode string is terminated with a Unicode
390 * NULL character.
391 *
392 * Side effects:
393 * None.
394 *
395 *---------------------------------------------------------------------------
396 */
397
398 Tcl_UniChar *
399 Tcl_UtfToUniCharDString(string, length, dsPtr)
400 CONST char *string; /* UTF-8 string to convert to Unicode. */
401 int length; /* Length of UTF-8 string in bytes, or -1
402 * for strlen(). */
403 Tcl_DString *dsPtr; /* Unicode representation of string is
404 * appended to this previously initialized
405 * DString. */
406 {
407 Tcl_UniChar *w, *wString;
408 CONST char *p, *end;
409 int oldLength;
410
411 if (length < 0) {
412 length = strlen(string);
413 }
414
415 /*
416 * Unicode string length in Tcl_UniChars will be <= UTF-8 string length
417 * in bytes.
418 */
419
420 oldLength = Tcl_DStringLength(dsPtr);
421 Tcl_DStringSetLength(dsPtr,
422 (int) ((oldLength + length + 1) * sizeof(Tcl_UniChar)));
423 wString = (Tcl_UniChar *) (Tcl_DStringValue(dsPtr) + oldLength);
424
425 w = wString;
426 end = string + length;
427 for (p = string; p < end; ) {
428 p += Tcl_UtfToUniChar(p, w);
429 w++;
430 }
431 *w = '\0';
432 Tcl_DStringSetLength(dsPtr,
433 (oldLength + ((char *) w - (char *) wString)));
434
435 return wString;
436 }
437
438 /*
439 *---------------------------------------------------------------------------
440 *
441 * Tcl_UtfCharComplete --
442 *
443 * Determine if the UTF-8 string of the given length is long enough
444 * to be decoded by Tcl_UtfToUniChar(). This does not ensure that the
445 * UTF-8 string is properly formed. Equivalent to Plan 9 fullrune().
446 *
447 * Results:
448 * The return value is 0 if the string is not long enough, non-zero
449 * otherwise.
450 *
451 * Side effects:
452 * None.
453 *
454 *---------------------------------------------------------------------------
455 */
456
457 int
458 Tcl_UtfCharComplete(str, len)
459 CONST char *str; /* String to check if first few bytes
460 * contain a complete UTF-8 character. */
461 int len; /* Length of above string in bytes. */
462 {
463 int ch;
464
465 ch = *((unsigned char *) str);
466 return len >= totalBytes[ch];
467 }
468
469 /*
470 *---------------------------------------------------------------------------
471 *
472 * Tcl_NumUtfChars --
473 *
474 * Returns the number of characters (not bytes) in the UTF-8 string,
475 * not including the terminating NULL byte. This is equivalent to
476 * Plan 9 utflen() and utfnlen().
477 *
478 * Results:
479 * As above.
480 *
481 * Side effects:
482 * None.
483 *
484 *---------------------------------------------------------------------------
485 */
486
487 int
488 Tcl_NumUtfChars(str, len)
489 register CONST char *str; /* The UTF-8 string to measure. */
490 int len; /* The length of the string in bytes, or -1
491 * for strlen(string). */
492 {
493 Tcl_UniChar ch;
494 register Tcl_UniChar *chPtr = &ch;
495 register int n;
496 int i;
497
498 /*
499 * The separate implementations are faster.
500 */
501
502 i = 0;
503 if (len < 0) {
504 while (1) {
505 str += Tcl_UtfToUniChar(str, chPtr);
506 if (ch == '\0') {
507 break;
508 }
509 i++;
510 }
511 } else {
512 while (len > 0) {
513 n = Tcl_UtfToUniChar(str, chPtr);
514 len -= n;
515 str += n;
516 i++;
517 }
518 }
519 return i;
520 }
521
522 /*
523 *---------------------------------------------------------------------------
524 *
525 * Tcl_UtfFindFirst --
526 *
527 * Returns a pointer to the first occurance of the given Tcl_UniChar
528 * in the NULL-terminated UTF-8 string. The NULL terminator is
529 * considered part of the UTF-8 string. Equivalent to Plan 9
530 * utfrune().
531 *
532 * Results:
533 * As above. If the Tcl_UniChar does not exist in the given string,
534 * the return value is NULL.
535 *
536 * Side effects:
537 * None.
538 *
539 *---------------------------------------------------------------------------
540 */
541 char *
542 Tcl_UtfFindFirst(string, ch)
543 CONST char *string; /* The UTF-8 string to be searched. */
544 int ch; /* The Tcl_UniChar to search for. */
545 {
546 int len;
547 Tcl_UniChar find;
548
549 while (1) {
550 len = Tcl_UtfToUniChar(string, &find);
551 if (find == ch) {
552 return (char *) string;
553 }
554 if (*string == '\0') {
555 return NULL;
556 }
557 string += len;
558 }
559 }
560
561 /*
562 *---------------------------------------------------------------------------
563 *
564 * Tcl_UtfFindLast --
565 *
566 * Returns a pointer to the last occurance of the given Tcl_UniChar
567 * in the NULL-terminated UTF-8 string. The NULL terminator is
568 * considered part of the UTF-8 string. Equivalent to Plan 9
569 * utfrrune().
570 *
571 * Results:
572 * As above. If the Tcl_UniChar does not exist in the given string,
573 * the return value is NULL.
574 *
575 * Side effects:
576 * None.
577 *
578 *---------------------------------------------------------------------------
579 */
580
581 char *
582 Tcl_UtfFindLast(string, ch)
583 CONST char *string; /* The UTF-8 string to be searched. */
584 int ch; /* The Tcl_UniChar to search for. */
585 {
586 int len;
587 Tcl_UniChar find;
588 CONST char *last;
589
590 last = NULL;
591 while (1) {
592 len = Tcl_UtfToUniChar(string, &find);
593 if (find == ch) {
594 last = string;
595 }
596 if (*string == '\0') {
597 break;
598 }
599 string += len;
600 }
601 return (char *) last;
602 }
603
604 /*
605 *---------------------------------------------------------------------------
606 *
607 * Tcl_UtfNext --
608 *
609 * Given a pointer to some current location in a UTF-8 string,
610 * move forward one character. The caller must ensure that they
611 * are not asking for the next character after the last character
612 * in the string.
613 *
614 * Results:
615 * The return value is the pointer to the next character in
616 * the UTF-8 string.
617 *
618 * Side effects:
619 * None.
620 *
621 *---------------------------------------------------------------------------
622 */
623
624 char *
625 Tcl_UtfNext(str)
626 CONST char *str; /* The current location in the string. */
627 {
628 Tcl_UniChar ch;
629
630 return (char *) str + Tcl_UtfToUniChar(str, &ch);
631 }
632
633 /*
634 *---------------------------------------------------------------------------
635 *
636 * Tcl_UtfPrev --
637 *
638 * Given a pointer to some current location in a UTF-8 string,
639 * move backwards one character.
640 *
641 * Results:
642 * The return value is a pointer to the previous character in the
643 * UTF-8 string. If the current location was already at the
644 * beginning of the string, the return value will also be a
645 * pointer to the beginning of the string.
646 *
647 * Side effects:
648 * None.
649 *
650 *---------------------------------------------------------------------------
651 */
652
653 char *
654 Tcl_UtfPrev(str, start)
655 CONST char *str; /* The current location in the string. */
656 CONST char *start; /* Pointer to the beginning of the
657 * string, to avoid going backwards too
658 * far. */
659 {
660 CONST char *look;
661 int i, byte;
662
663 str--;
664 look = str;
665 for (i = 0; i < TCL_UTF_MAX; i++) {
666 if (look < start) {
667 if (str < start) {
668 str = start;
669 }
670 break;
671 }
672 byte = *((unsigned char *) look);
673 if (byte < 0x80) {
674 break;
675 }
676 if (byte >= 0xC0) {
677 if (totalBytes[byte] != i + 1) {
678 break;
679 }
680 return (char *) look;
681 }
682 look--;
683 }
684 return (char *) str;
685 }
686
687 /*
688 *---------------------------------------------------------------------------
689 *
690 * Tcl_UniCharAtIndex --
691 *
692 * Returns the Unicode character represented at the specified
693 * character (not byte) position in the UTF-8 string.
694 *
695 * Results:
696 * As above.
697 *
698 * Side effects:
699 * None.
700 *
701 *---------------------------------------------------------------------------
702 */
703
704 Tcl_UniChar
705 Tcl_UniCharAtIndex(src, index)
706 register CONST char *src; /* The UTF-8 string to dereference. */
707 register int index; /* The position of the desired character. */
708 {
709 Tcl_UniChar ch;
710
711 while (index >= 0) {
712 index--;
713 src += Tcl_UtfToUniChar(src, &ch);
714 }
715 return ch;
716 }
717
718 /*
719 *---------------------------------------------------------------------------
720 *
721 * Tcl_UtfAtIndex --
722 *
723 * Returns a pointer to the specified character (not byte) position
724 * in the UTF-8 string.
725 *
726 * Results:
727 * As above.
728 *
729 * Side effects:
730 * None.
731 *
732 *---------------------------------------------------------------------------
733 */
734
735 char *
736 Tcl_UtfAtIndex(src, index)
737 register CONST char *src; /* The UTF-8 string. */
738 register int index; /* The position of the desired character. */
739 {
740 Tcl_UniChar ch;
741
742 while (index > 0) {
743 index--;
744 src += Tcl_UtfToUniChar(src, &ch);
745 }
746 return (char *) src;
747 }
748
749 /*
750 *---------------------------------------------------------------------------
751 *
752 * Tcl_UtfBackslash --
753 *
754 * Figure out how to handle a backslash sequence.
755 *
756 * Results:
757 * Stores the bytes represented by the backslash sequence in dst and
758 * returns the number of bytes written to dst. At most TCL_UTF_MAX
759 * bytes are written to dst; dst must have been large enough to accept
760 * those bytes. If readPtr isn't NULL then it is filled in with a
761 * count of the number of bytes in the backslash sequence.
762 *
763 * Side effects:
764 * The maximum number of bytes it takes to represent a Unicode
765 * character in UTF-8 is guaranteed to be less than the number of
766 * bytes used to express the backslash sequence that represents
767 * that Unicode character. If the target buffer into which the
768 * caller is going to store the bytes that represent the Unicode
769 * character is at least as large as the source buffer from which
770 * the backslashed sequence was extracted, no buffer overruns should
771 * occur.
772 *
773 *---------------------------------------------------------------------------
774 */
775
776 int
777 Tcl_UtfBackslash(src, readPtr, dst)
778 CONST char *src; /* Points to the backslash character of
779 * a backslash sequence. */
780 int *readPtr; /* Fill in with number of characters read
781 * from src, unless NULL. */
782 char *dst; /* Filled with the bytes represented by the
783 * backslash sequence. */
784 {
785 register CONST char *p = src+1;
786 int result, count, n;
787 char buf[TCL_UTF_MAX];
788
789 if (dst == NULL) {
790 dst = buf;
791 }
792
793 count = 2;
794 switch (*p) {
795 /*
796 * Note: in the conversions below, use absolute values (e.g.,
797 * 0xa) rather than symbolic values (e.g. \n) that get converted
798 * by the compiler. It's possible that compilers on some
799 * platforms will do the symbolic conversions differently, which
800 * could result in non-portable Tcl scripts.
801 */
802
803 case 'a':
804 result = 0x7;
805 break;
806 case 'b':
807 result = 0x8;
808 break;
809 case 'f':
810 result = 0xc;
811 break;
812 case 'n':
813 result = 0xa;
814 break;
815 case 'r':
816 result = 0xd;
817 break;
818 case 't':
819 result = 0x9;
820 break;
821 case 'v':
822 result = 0xb;
823 break;
824 case 'x':
825 if (isxdigit(UCHAR(p[1]))) { /* INTL: digit */
826 char *end;
827
828 result = (unsigned char) strtoul(p+1, &end, 16);
829 count = end - src;
830 } else {
831 count = 2;
832 result = 'x';
833 }
834 break;
835 case 'u':
836 result = 0;
837 for (count = 0; count < 4; count++) {
838 p++;
839 if (!isxdigit(UCHAR(*p))) { /* INTL: digit */
840 break;
841 }
842 n = *p - '0';
843 if (n > 9) {
844 n = n + '0' + 10 - 'A';
845 }
846 if (n > 16) {
847 n = n + 'A' - 'a';
848 }
849 result = (result << 4) + n;
850 }
851 if (count == 0) {
852 result = 'u';
853 }
854 count += 2;
855 break;
856
857 case '\n':
858 do {
859 p++;
860 } while ((*p == ' ') || (*p == '\t'));
861 result = ' ';
862 count = p - src;
863 break;
864 case 0:
865 result = '\\';
866 count = 1;
867 break;
868 default:
869 /*
870 * Check for an octal number \oo?o?
871 */
872 if (isdigit(UCHAR(*p)) && (UCHAR(*p) < '8')) { /* INTL: digit */
873 result = (unsigned char)(*p - '0');
874 p++;
875 if (!isdigit(UCHAR(*p)) || (UCHAR(*p) >= '8')) { /* INTL: digit */
876 break;
877 }
878 count = 3;
879 result = (unsigned char)((result << 3) + (*p - '0'));
880 p++;
881 if (!isdigit(UCHAR(*p)) || (UCHAR(*p) >= '8')) { /* INTL: digit */
882 break;
883 }
884 count = 4;
885 result = (unsigned char)((result << 3) + (*p - '0'));
886 break;
887 }
888 result = *p;
889 count = 2;
890 break;
891 }
892
893 if (readPtr != NULL) {
894 *readPtr = count;
895 }
896 return Tcl_UniCharToUtf(result, dst);
897 }
898
899 /*
900 *----------------------------------------------------------------------
901 *
902 * Tcl_UtfToUpper --
903 *
904 * Convert lowercase characters to uppercase characters in a UTF
905 * string in place. The conversion may shrink the UTF string.
906 *
907 * Results:
908 * Returns the number of bytes in the resulting string
909 * excluding the trailing null.
910 *
911 * Side effects:
912 * Writes a terminating null after the last converted character.
913 *
914 *----------------------------------------------------------------------
915 */
916
917 int
918 Tcl_UtfToUpper(str)
919 char *str; /* String to convert in place. */
920 {
921 Tcl_UniChar ch, upChar;
922 char *src, *dst;
923 int bytes;
924
925 /*
926 * Iterate over the string until we hit the terminating null.
927 */
928
929 src = dst = str;
930 while (*src) {
931 bytes = Tcl_UtfToUniChar(src, &ch);
932 upChar = Tcl_UniCharToUpper(ch);
933
934 /*
935 * To keep badly formed Utf strings from getting inflated by
936 * the conversion (thereby causing a segfault), only copy the
937 * upper case char to dst if its size is <= the original char.
938 */
939
940 if (bytes < UtfCount(upChar)) {
941 memcpy(dst, src, (size_t) bytes);
942 dst += bytes;
943 } else {
944 dst += Tcl_UniCharToUtf(upChar, dst);
945 }
946 src += bytes;
947 }
948 *dst = '\0';
949 return (dst - str);
950 }
951
952 /*
953 *----------------------------------------------------------------------
954 *
955 * Tcl_UtfToLower --
956 *
957 * Convert uppercase characters to lowercase characters in a UTF
958 * string in place. The conversion may shrink the UTF string.
959 *
960 * Results:
961 * Returns the number of bytes in the resulting string
962 * excluding the trailing null.
963 *
964 * Side effects:
965 * Writes a terminating null after the last converted character.
966 *
967 *----------------------------------------------------------------------
968 */
969
970 int
971 Tcl_UtfToLower(str)
972 char *str; /* String to convert in place. */
973 {
974 Tcl_UniChar ch, lowChar;
975 char *src, *dst;
976 int bytes;
977
978 /*
979 * Iterate over the string until we hit the terminating null.
980 */
981
982 src = dst = str;
983 while (*src) {
984 bytes = Tcl_UtfToUniChar(src, &ch);
985 lowChar = Tcl_UniCharToLower(ch);
986
987 /*
988 * To keep badly formed Utf strings from getting inflated by
989 * the conversion (thereby causing a segfault), only copy the
990 * lower case char to dst if its size is <= the original char.
991 */
992
993 if (bytes < UtfCount(lowChar)) {
994 memcpy(dst, src, (size_t) bytes);
995 dst += bytes;
996 } else {
997 dst += Tcl_UniCharToUtf(lowChar, dst);
998 }
999 src += bytes;
1000 }
1001 *dst = '\0';
1002 return (dst - str);
1003 }
1004
1005 /*
1006 *----------------------------------------------------------------------
1007 *
1008 * Tcl_UtfToTitle --
1009 *
1010 * Changes the first character of a UTF string to title case or
1011 * uppercase and the rest of the string to lowercase. The
1012 * conversion happens in place and may shrink the UTF string.
1013 *
1014 * Results:
1015 * Returns the number of bytes in the resulting string
1016 * excluding the trailing null.
1017 *
1018 * Side effects:
1019 * Writes a terminating null after the last converted character.
1020 *
1021 *----------------------------------------------------------------------
1022 */
1023
1024 int
1025 Tcl_UtfToTitle(str)
1026 char *str; /* String to convert in place. */
1027 {
1028 Tcl_UniChar ch, titleChar, lowChar;
1029 char *src, *dst;
1030 int bytes;
1031
1032 /*
1033 * Capitalize the first character and then lowercase the rest of the
1034 * characters until we get to a null.
1035 */
1036
1037 src = dst = str;
1038
1039 if (*src) {
1040 bytes = Tcl_UtfToUniChar(src, &ch);
1041 titleChar = Tcl_UniCharToTitle(ch);
1042
1043 if (bytes < UtfCount(titleChar)) {
1044 memcpy(dst, src, (size_t) bytes);
1045 dst += bytes;
1046 } else {
1047 dst += Tcl_UniCharToUtf(titleChar, dst);
1048 }
1049 src += bytes;
1050 }
1051 while (*src) {
1052 bytes = Tcl_UtfToUniChar(src, &ch);
1053 lowChar = Tcl_UniCharToLower(ch);
1054
1055 if (bytes < UtfCount(lowChar)) {
1056 memcpy(dst, src, (size_t) bytes);
1057 dst += bytes;
1058 } else {
1059 dst += Tcl_UniCharToUtf(lowChar, dst);
1060 }
1061 src += bytes;
1062 }
1063 *dst = '\0';
1064 return (dst - str);
1065 }
1066
1067 /*
1068 *----------------------------------------------------------------------
1069 *
1070 * Tcl_UtfNcmp --
1071 *
1072 * Compare at most n UTF chars of string cs to string ct. Both cs
1073 * and ct are assumed to be at least n UTF chars long.
1074 *
1075 * Results:
1076 * Return <0 if cs < ct, 0 if cs == ct, or >0 if cs > ct.
1077 *
1078 * Side effects:
1079 * None.
1080 *
1081 *----------------------------------------------------------------------
1082 */
1083
1084 int
1085 Tcl_UtfNcmp(cs, ct, n)
1086 CONST char *cs; /* UTF string to compare to ct. */
1087 CONST char *ct; /* UTF string cs is compared to. */
1088 unsigned long n; /* Number of UTF chars to compare. */
1089 {
1090 Tcl_UniChar ch1, ch2;
1091 /*
1092 * Another approach that should work is:
1093 * return memcmp(cs, ct, (unsigned) (Tcl_UtfAtIndex(cs, n) - cs));
1094 * That assumes that ct is a properly formed UTF, so we will just
1095 * be comparing the bytes that compromise those strings to the
1096 * char length n.
1097 */
1098 while (n-- > 0) {
1099 /*
1100 * n must be interpreted as chars, not bytes.
1101 * This should be called only when both strings are of
1102 * at least n chars long (no need for \0 check)
1103 */
1104 cs += Tcl_UtfToUniChar(cs, &ch1);
1105 ct += Tcl_UtfToUniChar(ct, &ch2);
1106 if (ch1 != ch2) {
1107 return (ch1 - ch2);
1108 }
1109 }
1110 return 0;
1111 }
1112
1113 /*
1114 *----------------------------------------------------------------------
1115 *
1116 * Tcl_UtfNcasecmp --
1117 *
1118 * Compare at most n UTF chars of string cs to string ct case
1119 * insensitive. Both cs and ct are assumed to be at least n
1120 * UTF chars long.
1121 *
1122 * Results:
1123 * Return <0 if cs < ct, 0 if cs == ct, or >0 if cs > ct.
1124 *
1125 * Side effects:
1126 * None.
1127 *
1128 *----------------------------------------------------------------------
1129 */
1130
1131 int
1132 Tcl_UtfNcasecmp(cs, ct, n)
1133 CONST char *cs; /* UTF string to compare to ct. */
1134 CONST char *ct; /* UTF string cs is compared to. */
1135 unsigned long n; /* Number of UTF chars to compare. */
1136 {
1137 Tcl_UniChar ch1, ch2;
1138 while (n-- > 0) {
1139 /*
1140 * n must be interpreted as chars, not bytes.
1141 * This should be called only when both strings are of
1142 * at least n chars long (no need for \0 check)
1143 */
1144 cs += Tcl_UtfToUniChar(cs, &ch1);
1145 ct += Tcl_UtfToUniChar(ct, &ch2);
1146 if (ch1 != ch2) {
1147 ch1 = Tcl_UniCharToLower(ch1);
1148 ch2 = Tcl_UniCharToLower(ch2);
1149 if (ch1 != ch2) {
1150 return (ch1 - ch2);
1151 }
1152 }
1153 }
1154 return 0;
1155 }
1156
1157 /*
1158 *----------------------------------------------------------------------
1159 *
1160 * Tcl_UniCharToUpper --
1161 *
1162 * Compute the uppercase equivalent of the given Unicode character.
1163 *
1164 * Results:
1165 * Returns the uppercase Unicode character.
1166 *
1167 * Side effects:
1168 * None.
1169 *
1170 *----------------------------------------------------------------------
1171 */
1172
1173 Tcl_UniChar
1174 Tcl_UniCharToUpper(ch)
1175 int ch; /* Unicode character to convert. */
1176 {
1177 int info = GetUniCharInfo(ch);
1178
1179 if (GetCaseType(info) & 0x04) {
1180 return (Tcl_UniChar) (ch - GetDelta(info));
1181 } else {
1182 return ch;
1183 }
1184 }
1185
1186 /*
1187 *----------------------------------------------------------------------
1188 *
1189 * Tcl_UniCharToLower --
1190 *
1191 * Compute the lowercase equivalent of the given Unicode character.
1192 *
1193 * Results:
1194 * Returns the lowercase Unicode character.
1195 *
1196 * Side effects:
1197 * None.
1198 *
1199 *----------------------------------------------------------------------
1200 */
1201
1202 Tcl_UniChar
1203 Tcl_UniCharToLower(ch)
1204 int ch; /* Unicode character to convert. */
1205 {
1206 int info = GetUniCharInfo(ch);
1207
1208 if (GetCaseType(info) & 0x02) {
1209 return (Tcl_UniChar) (ch + GetDelta(info));
1210 } else {
1211 return ch;
1212 }
1213 }
1214
1215 /*
1216 *----------------------------------------------------------------------
1217 *
1218 * Tcl_UniCharToTitle --
1219 *
1220 * Compute the titlecase equivalent of the given Unicode character.
1221 *
1222 * Results:
1223 * Returns the titlecase Unicode character.
1224 *
1225 * Side effects:
1226 * None.
1227 *
1228 *----------------------------------------------------------------------
1229 */
1230
1231 Tcl_UniChar
1232 Tcl_UniCharToTitle(ch)
1233 int ch; /* Unicode character to convert. */
1234 {
1235 int info = GetUniCharInfo(ch);
1236 int mode = GetCaseType(info);
1237
1238 if (mode & 0x1) {
1239 /*
1240 * Subtract or add one depending on the original case.
1241 */
1242
1243 return (Tcl_UniChar) (ch + ((mode & 0x4) ? -1 : 1));
1244 } else if (mode == 0x4) {
1245 return (Tcl_UniChar) (ch - GetDelta(info));
1246 } else {
1247 return ch;
1248 }
1249 }
1250
1251 /*
1252 *----------------------------------------------------------------------
1253 *
1254 * Tcl_UniCharLen --
1255 *
1256 * Find the length of a UniChar string. The str input must be null
1257 * terminated.
1258 *
1259 * Results:
1260 * Returns the length of str in UniChars (not bytes).
1261 *
1262 * Side effects:
1263 * None.
1264 *
1265 *----------------------------------------------------------------------
1266 */
1267
1268 int
1269 Tcl_UniCharLen(str)
1270 Tcl_UniChar *str; /* Unicode string to find length of. */
1271 {
1272 int len = 0;
1273
1274 while (*str != '\0') {
1275 len++;
1276 str++;
1277 }
1278 return len;
1279 }
1280
1281 /*
1282 *----------------------------------------------------------------------
1283 *
1284 * Tcl_UniCharNcmp --
1285 *
1286 * Compare at most n unichars of string cs to string ct. Both cs
1287 * and ct are assumed to be at least n unichars long.
1288 *
1289 * Results:
1290 * Return <0 if cs < ct, 0 if cs == ct, or >0 if cs > ct.
1291 *
1292 * Side effects:
1293 * None.
1294 *
1295 *----------------------------------------------------------------------
1296 */
1297
1298 int
1299 Tcl_UniCharNcmp(cs, ct, n)
1300 CONST Tcl_UniChar *cs; /* Unicode string to compare to ct. */
1301 CONST Tcl_UniChar *ct; /* Unicode string cs is compared to. */
1302 unsigned long n; /* Number of unichars to compare. */
1303 {
1304 for ( ; n != 0; n--, cs++, ct++) {
1305 if (*cs != *ct) {
1306 return *cs - *ct;
1307 }
1308 if (*cs == '\0') {
1309 break;
1310 }
1311 }
1312 return 0;
1313 }
1314
1315 /*
1316 *----------------------------------------------------------------------
1317 *
1318 * Tcl_UniCharIsAlnum --
1319 *
1320 * Test if a character is an alphanumeric Unicode character.
1321 *
1322 * Results:
1323 * Returns 1 if character is alphanumeric.
1324 *
1325 * Side effects:
1326 * None.
1327 *
1328 *----------------------------------------------------------------------
1329 */
1330
1331 int
1332 Tcl_UniCharIsAlnum(ch)
1333 int ch; /* Unicode character to test. */
1334 {
1335 register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1336
1337 return (((ALPHA_BITS | DIGIT_BITS) >> category) & 1);
1338 }
1339
1340 /*
1341 *----------------------------------------------------------------------
1342 *
1343 * Tcl_UniCharIsAlpha --
1344 *
1345 * Test if a character is an alphabetic Unicode character.
1346 *
1347 * Results:
1348 * Returns 1 if character is alphabetic.
1349 *
1350 * Side effects:
1351 * None.
1352 *
1353 *----------------------------------------------------------------------
1354 */
1355
1356 int
1357 Tcl_UniCharIsAlpha(ch)
1358 int ch; /* Unicode character to test. */
1359 {
1360 register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1361 return ((ALPHA_BITS >> category) & 1);
1362 }
1363
1364 /*
1365 *----------------------------------------------------------------------
1366 *
1367 * Tcl_UniCharIsControl --
1368 *
1369 * Test if a character is a Unicode control character.
1370 *
1371 * Results:
1372 * Returns non-zero if character is a control.
1373 *
1374 * Side effects:
1375 * None.
1376 *
1377 *----------------------------------------------------------------------
1378 */
1379
1380 int
1381 Tcl_UniCharIsControl(ch)
1382 int ch; /* Unicode character to test. */
1383 {
1384 return ((GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK) == CONTROL);
1385 }
1386
1387 /*
1388 *----------------------------------------------------------------------
1389 *
1390 * Tcl_UniCharIsDigit --
1391 *
1392 * Test if a character is a numeric Unicode character.
1393 *
1394 * Results:
1395 * Returns non-zero if character is a digit.
1396 *
1397 * Side effects:
1398 * None.
1399 *
1400 *----------------------------------------------------------------------
1401 */
1402
1403 int
1404 Tcl_UniCharIsDigit(ch)
1405 int ch; /* Unicode character to test. */
1406 {
1407 return ((GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK)
1408 == DECIMAL_DIGIT_NUMBER);
1409 }
1410
1411 /*
1412 *----------------------------------------------------------------------
1413 *
1414 * Tcl_UniCharIsGraph --
1415 *
1416 * Test if a character is any Unicode print character except space.
1417 *
1418 * Results:
1419 * Returns non-zero if character is printable, but not space.
1420 *
1421 * Side effects:
1422 * None.
1423 *
1424 *----------------------------------------------------------------------
1425 */
1426
1427 int
1428 Tcl_UniCharIsGraph(ch)
1429 int ch; /* Unicode character to test. */
1430 {
1431 register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1432 return (((PRINT_BITS >> category) & 1) && ((unsigned char) ch != ' '));
1433 }
1434
1435 /*
1436 *----------------------------------------------------------------------
1437 *
1438 * Tcl_UniCharIsLower --
1439 *
1440 * Test if a character is a lowercase Unicode character.
1441 *
1442 * Results:
1443 * Returns non-zero if character is lowercase.
1444 *
1445 * Side effects:
1446 * None.
1447 *
1448 *----------------------------------------------------------------------
1449 */
1450
1451 int
1452 Tcl_UniCharIsLower(ch)
1453 int ch; /* Unicode character to test. */
1454 {
1455 return ((GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK) == LOWERCASE_LETTER);
1456 }
1457
1458 /*
1459 *----------------------------------------------------------------------
1460 *
1461 * Tcl_UniCharIsPrint --
1462 *
1463 * Test if a character is a Unicode print character.
1464 *
1465 * Results:
1466 * Returns non-zero if character is printable.
1467 *
1468 * Side effects:
1469 * None.
1470 *
1471 *----------------------------------------------------------------------
1472 */
1473
1474 int
1475 Tcl_UniCharIsPrint(ch)
1476 int ch; /* Unicode character to test. */
1477 {
1478 register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1479 return ((PRINT_BITS >> category) & 1);
1480 }
1481
1482 /*
1483 *----------------------------------------------------------------------
1484 *
1485 * Tcl_UniCharIsPunct --
1486 *
1487 * Test if a character is a Unicode punctuation character.
1488 *
1489 * Results:
1490 * Returns non-zero if character is punct.
1491 *
1492 * Side effects:
1493 * None.
1494 *
1495 *----------------------------------------------------------------------
1496 */
1497
1498 int
1499 Tcl_UniCharIsPunct(ch)
1500 int ch; /* Unicode character to test. */
1501 {
1502 register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1503 return ((PUNCT_BITS >> category) & 1);
1504 }
1505
1506 /*
1507 *----------------------------------------------------------------------
1508 *
1509 * Tcl_UniCharIsSpace --
1510 *
1511 * Test if a character is a whitespace Unicode character.
1512 *
1513 * Results:
1514 * Returns non-zero if character is a space.
1515 *
1516 * Side effects:
1517 * None.
1518 *
1519 *----------------------------------------------------------------------
1520 */
1521
1522 int
1523 Tcl_UniCharIsSpace(ch)
1524 int ch; /* Unicode character to test. */
1525 {
1526 register int category;
1527
1528 /*
1529 * If the character is within the first 127 characters, just use the
1530 * standard C function, otherwise consult the Unicode table.
1531 */
1532
1533 if (ch < 0x80) {
1534 return isspace(UCHAR(ch)); /* INTL: ISO space */
1535 } else {
1536 category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1537 return ((SPACE_BITS >> category) & 1);
1538 }
1539 }
1540
1541 /*
1542 *----------------------------------------------------------------------
1543 *
1544 * Tcl_UniCharIsUpper --
1545 *
1546 * Test if a character is a uppercase Unicode character.
1547 *
1548 * Results:
1549 * Returns non-zero if character is uppercase.
1550 *
1551 * Side effects:
1552 * None.
1553 *
1554 *----------------------------------------------------------------------
1555 */
1556
1557 int
1558 Tcl_UniCharIsUpper(ch)
1559 int ch; /* Unicode character to test. */
1560 {
1561 return ((GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK) == UPPERCASE_LETTER);
1562 }
1563
1564 /*
1565 *----------------------------------------------------------------------
1566 *
1567 * Tcl_UniCharIsWordChar --
1568 *
1569 * Test if a character is alphanumeric or a connector punctuation
1570 * mark.
1571 *
1572 * Results:
1573 * Returns 1 if character is a word character.
1574 *
1575 * Side effects:
1576 * None.
1577 *
1578 *----------------------------------------------------------------------
1579 */
1580
1581 int
1582 Tcl_UniCharIsWordChar(ch)
1583 int ch; /* Unicode character to test. */
1584 {
1585 register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
1586
1587 return (((ALPHA_BITS | DIGIT_BITS | CONNECTOR_BITS) >> category) & 1);
1588 }
1589
1590
1591 /* $History: tclutf.c $
1592 *
1593 * ***************** Version 1 *****************
1594 * User: Dtashley Date: 1/02/01 Time: 1:05a
1595 * Created in $/IjuScripter, IjuConsole/Source/Tcl Base
1596 * Initial check-in.
1597 */
1598
1599 /* End of TCL_UTF.C */

dashley@gmail.com
ViewVC Help
Powered by ViewVC 1.1.25