1 /* $Id: icu_I18N.c,v 1.19 2007-05-22 07:51:45 adam Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
28 #include <yaz/timing.h>
41 #include <unicode/ustring.h> /* some more string fcns*/
42 #include <unicode/uchar.h> /* char names */
45 //#include <unicode/ustdio.h>
46 //#include <unicode/utypes.h> /* Basic ICU data types */
47 #include <unicode/ucol.h>
48 //#include <unicode/ucnv.h> /* C Converter API */
49 //#include <unicode/uloc.h>
50 //#include <unicode/ubrk.h>
51 /* #include <unicode/unistr.h> */
56 int icu_check_status (UErrorCode status)
58 if(U_FAILURE(status)){
60 "ICU: %d %s\n", status, u_errorName(status));
69 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
71 struct icu_buf_utf16 * buf16
72 = (struct icu_buf_utf16 *) malloc(sizeof(struct icu_buf_utf16));
79 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
80 buf16->utf16[0] = (UChar) 0;
81 buf16->utf16_cap = capacity;
87 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
92 if (0 == buf16->utf16)
93 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
96 = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity);
97 buf16->utf16[0] = (UChar) 0;
99 buf16->utf16_cap = capacity;
105 buf16->utf16_len = 0;
106 buf16->utf16_cap = 0;
114 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
115 struct icu_buf_utf16 * src16)
121 if (dest16->utf16_cap < src16->utf16_len)
122 icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
124 u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
125 dest16->utf16_len = src16->utf16_len;
131 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
145 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
147 struct icu_buf_utf8 * buf8
148 = (struct icu_buf_utf8 *) malloc(sizeof(struct icu_buf_utf8));
155 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
156 buf8->utf8[0] = (uint8_t) 0;
157 buf8->utf8_cap = capacity;
164 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
170 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
173 = (uint8_t *) realloc(buf8->utf8,
174 sizeof(uint8_t) * capacity);
175 buf8->utf8[0] = (uint8_t) 0;
177 buf8->utf8_cap = capacity;
192 struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8,
193 struct icu_buf_utf8 * src8)
200 if (dest8->utf8_cap < src8->utf8_len)
201 icu_buf_utf8_resize(dest8, src8->utf8_len * 2);
203 strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len);
210 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
221 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
222 struct icu_buf_utf8 * src8,
225 int32_t utf16_len = 0;
227 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
229 (const char *) src8->utf8, src8->utf8_len, status);
231 // check for buffer overflow, resize and retry
232 if (*status == U_BUFFER_OVERFLOW_ERROR
233 //|| dest16->utf16_len > dest16->utf16_cap
235 icu_buf_utf16_resize(dest16, utf16_len * 2);
236 *status = U_ZERO_ERROR;
237 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
239 (const char *) src8->utf8, src8->utf8_len, status);
242 //if (*status != U_BUFFER_OVERFLOW_ERROR
243 if (U_SUCCESS(*status)
244 && utf16_len < dest16->utf16_cap)
245 dest16->utf16_len = utf16_len;
247 dest16->utf16[0] = (UChar) 0;
248 dest16->utf16_len = 0;
256 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
257 const char * src8cstr,
260 size_t src8cstr_len = 0;
261 int32_t utf16_len = 0;
263 src8cstr_len = strlen(src8cstr);
265 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
267 src8cstr, src8cstr_len, status);
269 // check for buffer overflow, resize and retry
270 if (*status == U_BUFFER_OVERFLOW_ERROR
271 //|| dest16->utf16_len > dest16->utf16_cap
273 icu_buf_utf16_resize(dest16, utf16_len * 2);
274 *status = U_ZERO_ERROR;
275 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
277 src8cstr, src8cstr_len, status);
280 // if (*status != U_BUFFER_OVERFLOW_ERROR
281 if (U_SUCCESS(*status)
282 && utf16_len < dest16->utf16_cap)
283 dest16->utf16_len = utf16_len;
285 dest16->utf16[0] = (UChar) 0;
286 dest16->utf16_len = 0;
295 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
296 struct icu_buf_utf16 * src16,
299 int32_t utf8_len = 0;
301 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
303 src16->utf16, src16->utf16_len, status);
305 // check for buffer overflow, resize and retry
306 if (*status == U_BUFFER_OVERFLOW_ERROR
307 //|| dest8->utf8_len > dest8->utf8_cap
309 icu_buf_utf8_resize(dest8, utf8_len * 2);
310 *status = U_ZERO_ERROR;
311 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
313 src16->utf16, src16->utf16_len, status);
317 //if (*status != U_BUFFER_OVERFLOW_ERROR
318 if (U_SUCCESS(*status)
319 && utf8_len < dest8->utf8_cap)
320 dest8->utf8_len = utf8_len;
322 dest8->utf8[0] = (uint8_t) 0;
331 struct icu_casemap * icu_casemap_create(const char *locale, char action,
334 struct icu_casemap * casemap
335 = (struct icu_casemap *) malloc(sizeof(struct icu_casemap));
336 strcpy(casemap->locale, locale);
337 casemap->action = action;
339 switch(casemap->action) {
349 icu_casemap_destroy(casemap);
356 void icu_casemap_destroy(struct icu_casemap * casemap)
363 int icu_casemap_casemap(struct icu_casemap * casemap,
364 struct icu_buf_utf16 * dest16,
365 struct icu_buf_utf16 * src16,
371 return icu_utf16_casemap(dest16, src16,
372 casemap->locale, casemap->action, status);
376 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
377 struct icu_buf_utf16 * src16,
378 const char *locale, char action,
381 int32_t dest16_len = 0;
385 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
386 src16->utf16, src16->utf16_len,
390 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
391 src16->utf16, src16->utf16_len,
395 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
396 src16->utf16, src16->utf16_len,
400 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
401 src16->utf16, src16->utf16_len,
402 U_FOLD_CASE_DEFAULT, status);
406 return U_UNSUPPORTED_ERROR;
410 // check for buffer overflow, resize and retry
411 if (*status == U_BUFFER_OVERFLOW_ERROR
412 && dest16 != src16 // do not resize if in-place conversion
413 //|| dest16_len > dest16->utf16_cap
415 icu_buf_utf16_resize(dest16, dest16_len * 2);
416 *status = U_ZERO_ERROR;
421 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
422 src16->utf16, src16->utf16_len,
426 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
427 src16->utf16, src16->utf16_len,
431 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
432 src16->utf16, src16->utf16_len,
436 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
437 src16->utf16, src16->utf16_len,
438 U_FOLD_CASE_DEFAULT, status);
442 return U_UNSUPPORTED_ERROR;
447 if (U_SUCCESS(*status)
448 && dest16_len < dest16->utf16_cap)
449 dest16->utf16_len = dest16_len;
451 dest16->utf16[0] = (UChar) 0;
452 dest16->utf16_len = 0;
460 UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
461 struct icu_buf_utf8 * dest8,
462 struct icu_buf_utf16 * src16,
466 int32_t sortkey_len = 0;
468 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
469 dest8->utf8, dest8->utf8_cap);
471 // check for buffer overflow, resize and retry
472 if (sortkey_len > dest8->utf8_cap) {
473 icu_buf_utf8_resize(dest8, sortkey_len * 2);
474 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
475 dest8->utf8, dest8->utf8_cap);
478 if (U_SUCCESS(*status)
480 dest8->utf8_len = sortkey_len;
482 dest8->utf8[0] = (UChar) 0;
491 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
494 struct icu_tokenizer * tokenizer
495 = (struct icu_tokenizer *) malloc(sizeof(struct icu_tokenizer));
497 strcpy(tokenizer->locale, locale);
498 tokenizer->action = action;
500 tokenizer->buf16 = 0;
501 tokenizer->token_count = 0;
502 tokenizer->token_id = 0;
503 tokenizer->token_start = 0;
504 tokenizer->token_end = 0;
507 switch(tokenizer->action) {
510 = ubrk_open(UBRK_LINE, tokenizer->locale,
515 = ubrk_open(UBRK_SENTENCE, tokenizer->locale,
520 = ubrk_open(UBRK_WORD, tokenizer->locale,
525 = ubrk_open(UBRK_CHARACTER, tokenizer->locale,
530 = ubrk_open(UBRK_TITLE, tokenizer->locale,
534 *status = U_UNSUPPORTED_ERROR;
539 // ICU error stuff is a very funny business
540 if (U_SUCCESS(*status))
544 icu_tokenizer_destroy(tokenizer);
548 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
552 ubrk_close(tokenizer->bi);
557 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
558 struct icu_buf_utf16 * src16,
561 if (!tokenizer || !tokenizer->bi || !src16)
565 tokenizer->buf16 = src16;
566 tokenizer->token_count = 0;
567 tokenizer->token_id = 0;
568 tokenizer->token_start = 0;
569 tokenizer->token_end = 0;
571 ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
574 if (U_FAILURE(*status))
580 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
581 struct icu_buf_utf16 * tkn16,
584 int32_t tkn_start = 0;
589 if (!tokenizer || !tokenizer->bi
590 || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
593 // never change tokenizer->buf16 and keep always invariant
594 // 0 <= tokenizer->token_start
595 // <= tokenizer->token_end
596 // <= tokenizer->buf16->utf16_len
597 // returns length of token
599 if (0 == tokenizer->token_end) // first call
600 tkn_start = ubrk_first(tokenizer->bi);
601 else //successive calls
602 tkn_start = tokenizer->token_end;
605 tkn_end = ubrk_next(tokenizer->bi);
607 // repairing invariant at end of ubrk, which is UBRK_DONE = -1
608 if (UBRK_DONE == tkn_end)
609 tkn_end = tokenizer->buf16->utf16_len;
611 // copy out if everything is well
612 if(U_FAILURE(*status))
615 // everything OK, now update internal state
616 tkn_len = tkn_end - tkn_start;
619 tokenizer->token_count++;
620 tokenizer->token_id++;
622 tokenizer->token_id = 0;
624 tokenizer->token_start = tkn_start;
625 tokenizer->token_end = tkn_end;
628 // copying into token buffer if it exists
630 if (tkn16->utf16_cap < tkn_len)
631 icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
633 u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
636 tkn16->utf16_len = tkn_len;
643 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
645 return tokenizer->token_id;
648 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
650 return tokenizer->token_start;
653 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
655 return tokenizer->token_end;
658 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
660 return (tokenizer->token_end - tokenizer->token_start);
663 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
665 return tokenizer->token_count;
670 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
674 struct icu_normalizer * normalizer
675 = (struct icu_normalizer *) malloc(sizeof(struct icu_normalizer));
677 normalizer->action = action;
678 normalizer->trans = 0;
679 normalizer->rules16 = icu_buf_utf16_create(0);
680 icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
682 switch(normalizer->action) {
685 = utrans_openU(normalizer->rules16->utf16,
686 normalizer->rules16->utf16_len,
689 normalizer->parse_error, status);
690 // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans);
694 = utrans_openU(normalizer->rules16->utf16,
695 normalizer->rules16->utf16_len,
698 normalizer->parse_error, status);
699 // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans);
702 *status = U_UNSUPPORTED_ERROR;
707 if (U_SUCCESS(*status))
711 icu_normalizer_destroy(normalizer);
716 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
718 if (normalizer->rules16)
719 icu_buf_utf16_destroy(normalizer->rules16);
720 if (normalizer->trans)
722 // yaz_log(YLOG_LOG, "utrans_close %p", normalizer->trans);
723 utrans_close(normalizer->trans);
731 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
732 struct icu_buf_utf16 * dest16,
733 struct icu_buf_utf16 * src16,
736 if (!normalizer || !normalizer->trans || !src16 || !dest16)
739 if (!icu_buf_utf16_copy(dest16, src16))
742 utrans_transUChars (normalizer->trans,
743 dest16->utf16, &(dest16->utf16_len),
745 0, &(src16->utf16_len), status);
747 if (U_FAILURE(*status)){
748 dest16->utf16[0] = (UChar) 0;
749 dest16->utf16_len = 0;
752 return dest16->utf16_len;
758 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
759 enum icu_chain_step_type type,
760 const uint8_t * rule,
761 struct icu_buf_utf16 * buf16,
764 struct icu_chain_step * step = 0;
766 if(!chain || !type || !rule)
769 step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step));
772 step->more_tokens = 0;
773 step->need_new_token = 1;
780 // create auxilary objects
782 case ICU_chain_step_type_display:
784 case ICU_chain_step_type_norm:
786 case ICU_chain_step_type_sort:
788 case ICU_chain_step_type_casemap:
789 step->u.casemap = icu_casemap_create((char *) chain->locale,
790 (char) rule[0], status);
792 case ICU_chain_step_type_normalize:
793 step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
795 case ICU_chain_step_type_tokenize:
796 step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
797 (char) rule[0], status);
807 void icu_chain_step_destroy(struct icu_chain_step * step){
812 icu_chain_step_destroy(step->previous);
815 case ICU_chain_step_type_display:
817 case ICU_chain_step_type_norm:
819 case ICU_chain_step_type_sort:
821 case ICU_chain_step_type_casemap:
822 icu_casemap_destroy(step->u.casemap);
823 icu_buf_utf16_destroy(step->buf16);
825 case ICU_chain_step_type_normalize:
826 icu_normalizer_destroy(step->u.normalizer);
827 icu_buf_utf16_destroy(step->buf16);
829 case ICU_chain_step_type_tokenize:
830 icu_tokenizer_destroy(step->u.tokenizer);
831 icu_buf_utf16_destroy(step->buf16);
841 struct icu_chain * icu_chain_create(const uint8_t * identifier,
842 const uint8_t * locale)
845 struct icu_chain * chain
846 = (struct icu_chain *) malloc(sizeof(struct icu_chain));
848 strncpy((char *) chain->identifier, (const char *) identifier, 128);
849 chain->identifier[128 - 1] = '\0';
850 strncpy((char *) chain->locale, (const char *) locale, 16);
851 chain->locale[16 - 1] = '\0';
853 chain->token_count = 0;
855 chain->display8 = icu_buf_utf8_create(0);
856 chain->norm8 = icu_buf_utf8_create(0);
857 chain->sort8 = icu_buf_utf8_create(0);
859 chain->src16 = icu_buf_utf16_create(0);
867 void icu_chain_destroy(struct icu_chain * chain)
870 icu_buf_utf8_destroy(chain->display8);
871 icu_buf_utf8_destroy(chain->norm8);
872 icu_buf_utf8_destroy(chain->sort8);
874 icu_buf_utf16_destroy(chain->src16);
876 icu_chain_step_destroy(chain->steps);
883 struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
884 UErrorCode * status){
887 struct icu_chain * chain = 0;
890 ||xml_node->type != XML_ELEMENT_NODE
891 || strcmp((const char *) xml_node->name, "icu_chain"))
895 xmlChar *xml_id = xmlGetProp(xml_node, (xmlChar *) "id");
896 xmlChar *xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale");
898 if (!xml_id || !strlen((const char *) xml_id)
899 || !xml_locale || !strlen((const char *) xml_locale))
902 chain = icu_chain_create((const uint8_t *) xml_id,
903 (const uint8_t *) xml_locale);
910 for (node = xml_node->children; node; node = node->next)
912 if (node->type != XML_ELEMENT_NODE)
915 xmlChar *xml_rule = xmlGetProp(node, (xmlChar *) "rule");
916 struct icu_chain_step * step = 0;
918 if (!strcmp((const char *) node->name,
919 (const char *) "casemap")){
920 step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
921 (const uint8_t *) xml_rule, status);
923 else if (!strcmp((const char *) node->name,
924 (const char *) "normalize")){
925 step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
926 (const uint8_t *) xml_rule, status);
928 else if (!strcmp((const char *) node->name,
929 (const char *) "tokenize")){
930 step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
931 (const uint8_t *) xml_rule, status);
933 else if (!strcmp((const char *) node->name,
934 (const char *) "display")){
935 step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
936 (const uint8_t *) "", status);
938 else if (!strcmp((const char *) node->name,
939 (const char *) "normal")){
940 step = icu_chain_insert_step(chain, ICU_chain_step_type_norm,
941 (const uint8_t *) "", status);
943 else if (!strcmp((const char *) node->name,
944 (const char *) "sort")){
945 step = icu_chain_insert_step(chain, ICU_chain_step_type_sort,
946 (const uint8_t *) "", status);
950 if (!step || U_FAILURE(*status)){
951 icu_chain_destroy(chain);
963 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
964 enum icu_chain_step_type type,
965 const uint8_t * rule,
968 struct icu_chain_step * step = 0;
969 struct icu_buf_utf16 * src16 = 0;
970 struct icu_buf_utf16 * buf16 = 0;
972 if (!chain || !type || !rule)
975 // assign utf16 src buffers as needed
976 if (chain->steps && chain->steps->buf16)
977 src16 = chain->steps->buf16;
978 else if (chain->src16)
979 src16 = chain->src16;
984 // create utf16 destination buffers as needed, or
986 case ICU_chain_step_type_display:
989 case ICU_chain_step_type_norm:
992 case ICU_chain_step_type_sort:
995 case ICU_chain_step_type_casemap:
996 buf16 = icu_buf_utf16_create(0);
998 case ICU_chain_step_type_normalize:
999 buf16 = icu_buf_utf16_create(0);
1001 case ICU_chain_step_type_tokenize:
1002 buf16 = icu_buf_utf16_create(0);
1008 // create actual chain step with this buffer
1009 step = icu_chain_step_create(chain, type, rule, buf16, status);
1011 step->previous = chain->steps;
1012 chain->steps = step;
1018 int icu_chain_step_next_token(struct icu_chain * chain,
1019 struct icu_chain_step * step,
1022 struct icu_buf_utf16 * src16 = 0;
1024 //printf("icu_chain_step_next_token %d\n", (int) step);
1026 if (!chain || !chain->src16 || !step || !step->more_tokens)
1029 // assign utf16 src buffers as neeed, advance in previous steps
1030 // tokens until non-zero token met, and setting stop condition
1031 if (step->previous){
1032 src16 = step->previous->buf16;
1033 if (step->need_new_token)
1034 //while (step->more_tokens && !src16->utf16_len)
1036 = icu_chain_step_next_token(chain, step->previous, status);
1038 else { // first step can only work once on chain->src16 input buffer
1039 src16 = chain->src16;
1040 step->more_tokens = 1;
1043 // stop if nothing to process
1044 // i.e new token source was not properly assigned
1045 if (!step->more_tokens || !src16) // || !src16->utf16_len
1048 //printf("icu_chain_step_next_token %d working\n", (int) step);
1051 // perform the work, eventually put this steps output in
1052 // step->buf16 or the chains UTF8 output buffers
1053 switch(step->type) {
1054 case ICU_chain_step_type_display:
1055 icu_utf16_to_utf8(chain->display8, src16, status);
1057 case ICU_chain_step_type_norm:
1058 icu_utf16_to_utf8(chain->norm8, src16, status);
1060 case ICU_chain_step_type_sort:
1061 icu_utf16_to_utf8(chain->sort8, src16, status);
1063 case ICU_chain_step_type_casemap:
1064 icu_casemap_casemap(step->u.casemap,
1065 step->buf16, src16, status);
1067 case ICU_chain_step_type_normalize:
1068 icu_normalizer_normalize(step->u.normalizer,
1069 step->buf16, src16, status);
1071 case ICU_chain_step_type_tokenize:
1072 // attach to new src16 token only first time during splitting
1073 if (step->need_new_token){
1074 icu_tokenizer_attach(step->u.tokenizer, src16, status);
1075 step->need_new_token = 0;
1077 // splitting one src16 token into multiple buf16 tokens
1079 = icu_tokenizer_next_token(step->u.tokenizer,
1080 step->buf16, status);
1081 // make sure to get new previous token if this one had been used up
1082 if (step->previous && !step->more_tokens){
1083 if (icu_chain_step_next_token(chain, step->previous, status)){
1084 icu_tokenizer_attach(step->u.tokenizer, src16, status);
1085 step->need_new_token = 0;
1087 = icu_tokenizer_next_token(step->u.tokenizer,
1088 step->buf16, status);
1091 if (0 == step->more_tokens)
1101 // stop further token processing if last step and
1102 // new tokens are needed from previous (non-existing) step
1103 if (!step->previous && step->need_new_token)
1104 step->more_tokens = 0;
1106 //printf("%d %d %d\n",
1107 // step->more_tokens, src16->utf16_len, step->buf16->utf16_len);
1110 if (U_FAILURE(*status))
1118 int icu_chain_assign_cstr(struct icu_chain * chain,
1119 const char * src8cstr,
1122 struct icu_chain_step * stp = 0;
1124 if (!chain || !src8cstr)
1129 // clear token count
1130 chain->token_count = 0;
1132 // clear all steps stop states
1135 stp->more_tokens = 1;
1136 stp = stp->previous;
1139 // finally convert UTF8 to UTF16 string
1140 icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status);
1142 if (U_FAILURE(*status))
1150 int icu_chain_next_token(struct icu_chain * chain,
1155 if (!chain || !chain->steps)
1158 success = icu_chain_step_next_token(chain, chain->steps, status);
1161 chain->token_count++;
1162 return chain->token_count;
1168 int icu_chain_get_token_count(struct icu_chain * chain)
1173 return chain->token_count;
1178 const char * icu_chain_get_display(struct icu_chain * chain)
1180 if (chain->display8)
1181 return (const char *) chain->display8->utf8;
1186 const char * icu_chain_get_norm(struct icu_chain * chain)
1189 return (const char *) chain->norm8->utf8;
1194 const char * icu_chain_get_sort(struct icu_chain * chain)
1197 return (const char *) chain->sort8->utf8;
1213 * indent-tabs-mode: nil
1215 * vim: shiftwidth=4 tabstop=8 expandtab