1 /* This file is part of Pazpar2.
2 Copyright (C) 2006-2008 Index Data
4 Pazpar2 is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26 #include <yaz/timing.h>
39 #include <unicode/ustring.h> /* some more string fcns*/
40 #include <unicode/uchar.h> /* char names */
43 //#include <unicode/ustdio.h>
44 //#include <unicode/utypes.h> /* Basic ICU data types */
45 #include <unicode/ucol.h>
46 //#include <unicode/ucnv.h> /* C Converter API */
47 //#include <unicode/uloc.h>
48 //#include <unicode/ubrk.h>
49 /* #include <unicode/unistr.h> */
54 int icu_check_status (UErrorCode status)
56 if(U_FAILURE(status)){
58 "ICU: %d %s\n", status, u_errorName(status));
67 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
69 struct icu_buf_utf16 * buf16
70 = (struct icu_buf_utf16 *) malloc(sizeof(struct icu_buf_utf16));
77 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
78 buf16->utf16[0] = (UChar) 0;
79 buf16->utf16_cap = capacity;
84 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
89 if (0 == buf16->utf16)
90 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
93 = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity);
94 buf16->utf16[0] = (UChar) 0;
96 buf16->utf16_cap = capacity;
102 buf16->utf16_len = 0;
103 buf16->utf16_cap = 0;
111 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
112 struct icu_buf_utf16 * src16)
118 if (dest16->utf16_cap < src16->utf16_len)
119 icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
121 u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
122 dest16->utf16_len = src16->utf16_len;
128 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
142 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
144 struct icu_buf_utf8 * buf8
145 = (struct icu_buf_utf8 *) malloc(sizeof(struct icu_buf_utf8));
152 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
153 buf8->utf8[0] = (uint8_t) 0;
154 buf8->utf8_cap = capacity;
161 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
167 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
170 = (uint8_t *) realloc(buf8->utf8,
171 sizeof(uint8_t) * capacity);
172 buf8->utf8_cap = capacity;
187 struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8,
188 struct icu_buf_utf8 * src8)
195 if (dest8->utf8_cap < src8->utf8_len)
196 icu_buf_utf8_resize(dest8, src8->utf8_len * 2);
198 strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len);
204 const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
206 if (!src8 || src8->utf8_len == 0)
208 if (src8->utf8_len == src8->utf8_cap)
209 src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1);
210 src8->utf8[src8->utf8_len] = '\0';
211 return (const char *) src8->utf8;
215 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
226 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
227 struct icu_buf_utf8 * src8,
230 int32_t utf16_len = 0;
232 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
234 (const char *) src8->utf8, src8->utf8_len, status);
236 // check for buffer overflow, resize and retry
237 if (*status == U_BUFFER_OVERFLOW_ERROR
238 //|| dest16->utf16_len > dest16->utf16_cap
240 icu_buf_utf16_resize(dest16, utf16_len * 2);
241 *status = U_ZERO_ERROR;
242 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
244 (const char *) src8->utf8, src8->utf8_len, status);
247 //if (*status != U_BUFFER_OVERFLOW_ERROR
248 if (U_SUCCESS(*status)
249 && utf16_len <= dest16->utf16_cap)
250 dest16->utf16_len = utf16_len;
252 dest16->utf16[0] = (UChar) 0;
253 dest16->utf16_len = 0;
261 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
262 const char * src8cstr,
265 size_t src8cstr_len = 0;
266 int32_t utf16_len = 0;
268 src8cstr_len = strlen(src8cstr);
270 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
272 src8cstr, src8cstr_len, status);
274 // check for buffer overflow, resize and retry
275 if (*status == U_BUFFER_OVERFLOW_ERROR
276 //|| dest16->utf16_len > dest16->utf16_cap
278 icu_buf_utf16_resize(dest16, utf16_len * 2);
279 *status = U_ZERO_ERROR;
280 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
282 src8cstr, src8cstr_len, status);
285 // if (*status != U_BUFFER_OVERFLOW_ERROR
286 if (U_SUCCESS(*status)
287 && utf16_len <= dest16->utf16_cap)
288 dest16->utf16_len = utf16_len;
290 dest16->utf16[0] = (UChar) 0;
291 dest16->utf16_len = 0;
300 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
301 struct icu_buf_utf16 * src16,
304 int32_t utf8_len = 0;
306 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
308 src16->utf16, src16->utf16_len, status);
310 // check for buffer overflow, resize and retry
311 if (*status == U_BUFFER_OVERFLOW_ERROR
312 //|| dest8->utf8_len > dest8->utf8_cap
314 icu_buf_utf8_resize(dest8, utf8_len * 2);
315 *status = U_ZERO_ERROR;
316 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
318 src16->utf16, src16->utf16_len, status);
322 //if (*status != U_BUFFER_OVERFLOW_ERROR
323 if (U_SUCCESS(*status)
324 && utf8_len <= dest8->utf8_cap)
325 dest8->utf8_len = utf8_len;
327 dest8->utf8[0] = (uint8_t) 0;
336 struct icu_casemap * icu_casemap_create(const char *locale, char action,
339 struct icu_casemap * casemap
340 = (struct icu_casemap *) malloc(sizeof(struct icu_casemap));
341 strcpy(casemap->locale, locale);
342 casemap->action = action;
344 switch(casemap->action) {
354 icu_casemap_destroy(casemap);
361 void icu_casemap_destroy(struct icu_casemap * casemap)
368 int icu_casemap_casemap(struct icu_casemap * casemap,
369 struct icu_buf_utf16 * dest16,
370 struct icu_buf_utf16 * src16,
376 return icu_utf16_casemap(dest16, src16,
377 casemap->locale, casemap->action, status);
381 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
382 struct icu_buf_utf16 * src16,
383 const char *locale, char action,
386 int32_t dest16_len = 0;
390 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
391 src16->utf16, src16->utf16_len,
395 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
396 src16->utf16, src16->utf16_len,
400 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
401 src16->utf16, src16->utf16_len,
405 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
406 src16->utf16, src16->utf16_len,
407 U_FOLD_CASE_DEFAULT, status);
411 return U_UNSUPPORTED_ERROR;
415 // check for buffer overflow, resize and retry
416 if (*status == U_BUFFER_OVERFLOW_ERROR
417 && dest16 != src16 // do not resize if in-place conversion
418 //|| dest16_len > dest16->utf16_cap
420 icu_buf_utf16_resize(dest16, dest16_len * 2);
421 *status = U_ZERO_ERROR;
426 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
427 src16->utf16, src16->utf16_len,
431 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
432 src16->utf16, src16->utf16_len,
436 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
437 src16->utf16, src16->utf16_len,
441 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
442 src16->utf16, src16->utf16_len,
443 U_FOLD_CASE_DEFAULT, status);
447 return U_UNSUPPORTED_ERROR;
452 if (U_SUCCESS(*status)
453 && dest16_len <= dest16->utf16_cap)
454 dest16->utf16_len = dest16_len;
456 dest16->utf16[0] = (UChar) 0;
457 dest16->utf16_len = 0;
465 UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
466 struct icu_buf_utf8 * dest8,
467 struct icu_buf_utf16 * src16,
471 int32_t sortkey_len = 0;
473 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
474 dest8->utf8, dest8->utf8_cap);
476 // check for buffer overflow, resize and retry
477 if (sortkey_len > dest8->utf8_cap) {
478 icu_buf_utf8_resize(dest8, sortkey_len * 2);
479 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
480 dest8->utf8, dest8->utf8_cap);
483 if (U_SUCCESS(*status)
485 dest8->utf8_len = sortkey_len;
487 dest8->utf8[0] = (UChar) 0;
496 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
499 struct icu_tokenizer * tokenizer
500 = (struct icu_tokenizer *) malloc(sizeof(struct icu_tokenizer));
502 strcpy(tokenizer->locale, locale);
503 tokenizer->action = action;
505 tokenizer->buf16 = 0;
506 tokenizer->token_count = 0;
507 tokenizer->token_id = 0;
508 tokenizer->token_start = 0;
509 tokenizer->token_end = 0;
512 switch(tokenizer->action) {
515 = ubrk_open(UBRK_LINE, tokenizer->locale,
520 = ubrk_open(UBRK_SENTENCE, tokenizer->locale,
525 = ubrk_open(UBRK_WORD, tokenizer->locale,
530 = ubrk_open(UBRK_CHARACTER, tokenizer->locale,
535 = ubrk_open(UBRK_TITLE, tokenizer->locale,
539 *status = U_UNSUPPORTED_ERROR;
544 // ICU error stuff is a very funny business
545 if (U_SUCCESS(*status))
549 icu_tokenizer_destroy(tokenizer);
553 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
557 ubrk_close(tokenizer->bi);
562 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
563 struct icu_buf_utf16 * src16,
566 if (!tokenizer || !tokenizer->bi || !src16)
570 tokenizer->buf16 = src16;
571 tokenizer->token_count = 0;
572 tokenizer->token_id = 0;
573 tokenizer->token_start = 0;
574 tokenizer->token_end = 0;
576 ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
579 if (U_FAILURE(*status))
585 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
586 struct icu_buf_utf16 * tkn16,
589 int32_t tkn_start = 0;
594 if (!tokenizer || !tokenizer->bi
595 || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
598 // never change tokenizer->buf16 and keep always invariant
599 // 0 <= tokenizer->token_start
600 // <= tokenizer->token_end
601 // <= tokenizer->buf16->utf16_len
602 // returns length of token
604 if (0 == tokenizer->token_end) // first call
605 tkn_start = ubrk_first(tokenizer->bi);
606 else //successive calls
607 tkn_start = tokenizer->token_end;
610 tkn_end = ubrk_next(tokenizer->bi);
612 // repairing invariant at end of ubrk, which is UBRK_DONE = -1
613 if (UBRK_DONE == tkn_end)
614 tkn_end = tokenizer->buf16->utf16_len;
616 // copy out if everything is well
617 if(U_FAILURE(*status))
620 // everything OK, now update internal state
621 tkn_len = tkn_end - tkn_start;
624 tokenizer->token_count++;
625 tokenizer->token_id++;
627 tokenizer->token_id = 0;
629 tokenizer->token_start = tkn_start;
630 tokenizer->token_end = tkn_end;
633 // copying into token buffer if it exists
635 if (tkn16->utf16_cap < tkn_len)
636 icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
638 u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
641 tkn16->utf16_len = tkn_len;
648 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
650 return tokenizer->token_id;
653 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
655 return tokenizer->token_start;
658 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
660 return tokenizer->token_end;
663 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
665 return (tokenizer->token_end - tokenizer->token_start);
668 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
670 return tokenizer->token_count;
675 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
679 struct icu_normalizer * normalizer
680 = (struct icu_normalizer *) malloc(sizeof(struct icu_normalizer));
682 normalizer->action = action;
683 normalizer->trans = 0;
684 normalizer->rules16 = icu_buf_utf16_create(0);
685 icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
687 switch(normalizer->action) {
690 = utrans_openU(normalizer->rules16->utf16,
691 normalizer->rules16->utf16_len,
694 normalizer->parse_error, status);
695 // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans);
699 = utrans_openU(normalizer->rules16->utf16,
700 normalizer->rules16->utf16_len,
703 normalizer->parse_error, status);
704 // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans);
707 *status = U_UNSUPPORTED_ERROR;
712 if (U_SUCCESS(*status))
716 icu_normalizer_destroy(normalizer);
721 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
723 if (normalizer->rules16)
724 icu_buf_utf16_destroy(normalizer->rules16);
725 if (normalizer->trans)
727 // yaz_log(YLOG_LOG, "utrans_close %p", normalizer->trans);
728 utrans_close(normalizer->trans);
736 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
737 struct icu_buf_utf16 * dest16,
738 struct icu_buf_utf16 * src16,
741 if (!normalizer || !normalizer->trans || !src16 || !dest16)
744 if (!icu_buf_utf16_copy(dest16, src16))
747 utrans_transUChars (normalizer->trans,
748 dest16->utf16, &(dest16->utf16_len),
750 0, &(src16->utf16_len), status);
752 if (U_FAILURE(*status)){
753 dest16->utf16[0] = (UChar) 0;
754 dest16->utf16_len = 0;
757 return dest16->utf16_len;
763 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
764 enum icu_chain_step_type type,
765 const uint8_t * rule,
766 struct icu_buf_utf16 * buf16,
769 struct icu_chain_step * step = 0;
771 if(!chain || !type || !rule)
774 step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step));
780 // create auxilary objects
782 case ICU_chain_step_type_display:
784 case ICU_chain_step_type_index:
786 case ICU_chain_step_type_sortkey:
788 case ICU_chain_step_type_casemap:
789 step->u.casemap = icu_casemap_create((char *) chain->locale,
790 (char) rule[0], status);
792 case ICU_chain_step_type_normalize:
793 step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
795 case ICU_chain_step_type_tokenize:
796 step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
797 (char) rule[0], status);
807 void icu_chain_step_destroy(struct icu_chain_step * step){
812 icu_chain_step_destroy(step->previous);
815 case ICU_chain_step_type_display:
817 case ICU_chain_step_type_index:
819 case ICU_chain_step_type_sortkey:
821 case ICU_chain_step_type_casemap:
822 icu_casemap_destroy(step->u.casemap);
823 icu_buf_utf16_destroy(step->buf16);
825 case ICU_chain_step_type_normalize:
826 icu_normalizer_destroy(step->u.normalizer);
827 icu_buf_utf16_destroy(step->buf16);
829 case ICU_chain_step_type_tokenize:
830 icu_tokenizer_destroy(step->u.tokenizer);
831 icu_buf_utf16_destroy(step->buf16);
841 struct icu_chain * icu_chain_create(const uint8_t * identifier,
842 const uint8_t * locale)
845 struct icu_chain * chain
846 = (struct icu_chain *) malloc(sizeof(struct icu_chain));
848 strncpy((char *) chain->identifier, (const char *) identifier, 128);
849 chain->identifier[128 - 1] = '\0';
850 strncpy((char *) chain->locale, (const char *) locale, 16);
851 chain->locale[16 - 1] = '\0';
853 chain->token_count = 0;
855 chain->display8 = icu_buf_utf8_create(0);
856 chain->norm8 = icu_buf_utf8_create(0);
857 chain->sort8 = icu_buf_utf8_create(0);
859 chain->src16 = icu_buf_utf16_create(0);
867 void icu_chain_destroy(struct icu_chain * chain)
870 icu_buf_utf8_destroy(chain->display8);
871 icu_buf_utf8_destroy(chain->norm8);
872 icu_buf_utf8_destroy(chain->sort8);
874 icu_buf_utf16_destroy(chain->src16);
876 icu_chain_step_destroy(chain->steps);
883 struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
884 UErrorCode * status){
887 struct icu_chain * chain = 0;
889 xmlChar *xml_locale = 0;
892 ||xml_node->type != XML_ELEMENT_NODE
893 || strcmp((const char *) xml_node->name, "icu_chain"))
897 xml_id = xmlGetProp(xml_node, (xmlChar *) "id");
898 xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale");
900 if (!xml_id || !strlen((const char *) xml_id)
901 || !xml_locale || !strlen((const char *) xml_locale))
904 chain = icu_chain_create((const uint8_t *) xml_id,
905 (const uint8_t *) xml_locale);
912 for (node = xml_node->children; node; node = node->next)
914 xmlChar *xml_rule = 0;
915 struct icu_chain_step * step = 0;
916 if (node->type != XML_ELEMENT_NODE)
919 xml_rule = xmlGetProp(node, (xmlChar *) "rule");
921 if (!strcmp((const char *) node->name,
922 (const char *) "casemap")){
923 step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
924 (const uint8_t *) xml_rule, status);
926 else if (!strcmp((const char *) node->name,
927 (const char *) "normalize")){
928 step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
929 (const uint8_t *) xml_rule, status);
931 else if (!strcmp((const char *) node->name,
932 (const char *) "tokenize")){
933 step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
934 (const uint8_t *) xml_rule, status);
936 else if (!strcmp((const char *) node->name,
937 (const char *) "display")){
938 step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
939 (const uint8_t *) "", status);
941 else if (!strcmp((const char *) node->name,
942 (const char *) "index")){
943 step = icu_chain_insert_step(chain, ICU_chain_step_type_index,
944 (const uint8_t *) "", status);
946 else if (!strcmp((const char *) node->name,
947 (const char *) "sortkey")){
948 step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey,
949 (const uint8_t *) "", status);
953 if (!step || U_FAILURE(*status)){
954 icu_chain_destroy(chain);
966 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
967 enum icu_chain_step_type type,
968 const uint8_t * rule,
971 struct icu_chain_step * step = 0;
972 struct icu_buf_utf16 * src16 = 0;
973 struct icu_buf_utf16 * buf16 = 0;
975 if (!chain || !type || !rule)
978 // assign utf16 src buffers as needed
979 if (chain->steps && chain->steps->buf16)
980 src16 = chain->steps->buf16;
981 else if (chain->src16)
982 src16 = chain->src16;
987 // create utf16 destination buffers as needed, or
989 case ICU_chain_step_type_display:
992 case ICU_chain_step_type_index:
995 case ICU_chain_step_type_sortkey:
998 case ICU_chain_step_type_casemap:
999 buf16 = icu_buf_utf16_create(0);
1001 case ICU_chain_step_type_normalize:
1002 buf16 = icu_buf_utf16_create(0);
1004 case ICU_chain_step_type_tokenize:
1005 buf16 = icu_buf_utf16_create(0);
1011 // create actual chain step with this buffer
1012 step = icu_chain_step_create(chain, type, rule, buf16, status);
1014 step->previous = chain->steps;
1015 chain->steps = step;
1021 int icu_chain_step_next_token(struct icu_chain * chain,
1022 struct icu_chain_step * step,
1025 struct icu_buf_utf16 * src16 = 0;
1027 //printf("icu_chain_step_next_token %d\n", (int) step);
1029 if (!chain || !chain->src16 || !step || !step->more_tokens)
1032 // assign utf16 src buffers as neeed, advance in previous steps
1033 // tokens until non-zero token met, and setting stop condition
1034 if (step->previous){
1035 src16 = step->previous->buf16;
1036 if (step->need_new_token)
1037 //while (step->more_tokens && !src16->utf16_len)
1039 = icu_chain_step_next_token(chain, step->previous, status);
1041 else { // first step can only work once on chain->src16 input buffer
1042 src16 = chain->src16;
1043 step->more_tokens = 1;
1046 // stop if nothing to process
1047 // i.e new token source was not properly assigned
1048 if (!step->more_tokens || !src16) // || !src16->utf16_len
1051 //printf("icu_chain_step_next_token %d working\n", (int) step);
1054 // perform the work, eventually put this steps output in
1055 // step->buf16 or the chains UTF8 output buffers
1056 switch(step->type) {
1057 case ICU_chain_step_type_display:
1058 icu_utf16_to_utf8(chain->display8, src16, status);
1060 case ICU_chain_step_type_index:
1061 icu_utf16_to_utf8(chain->norm8, src16, status);
1063 case ICU_chain_step_type_sortkey:
1064 icu_utf16_to_utf8(chain->sort8, src16, status);
1066 case ICU_chain_step_type_casemap:
1067 icu_casemap_casemap(step->u.casemap,
1068 step->buf16, src16, status);
1070 case ICU_chain_step_type_normalize:
1071 icu_normalizer_normalize(step->u.normalizer,
1072 step->buf16, src16, status);
1074 case ICU_chain_step_type_tokenize:
1075 // attach to new src16 token only first time during splitting
1076 if (step->need_new_token){
1077 icu_tokenizer_attach(step->u.tokenizer, src16, status);
1078 step->need_new_token = 0;
1080 // splitting one src16 token into multiple buf16 tokens
1082 = icu_tokenizer_next_token(step->u.tokenizer,
1083 step->buf16, status);
1084 // make sure to get new previous token if this one had been used up
1085 if (step->previous && !step->more_tokens){
1086 if (icu_chain_step_next_token(chain, step->previous, status)){
1087 icu_tokenizer_attach(step->u.tokenizer, src16, status);
1088 step->need_new_token = 0;
1090 = icu_tokenizer_next_token(step->u.tokenizer,
1091 step->buf16, status);
1094 if (0 == step->more_tokens)
1104 // stop further token processing if last step and
1105 // new tokens are needed from previous (non-existing) step
1106 if (!step->previous && step->need_new_token)
1107 step->more_tokens = 0;
1109 //printf("%d %d %d\n",
1110 // step->more_tokens, src16->utf16_len, step->buf16->utf16_len);
1113 if (U_FAILURE(*status))
1121 int icu_chain_assign_cstr(struct icu_chain * chain,
1122 const char * src8cstr,
1125 struct icu_chain_step * stp = 0;
1127 if (!chain || !src8cstr)
1132 // clear token count
1133 chain->token_count = 0;
1135 // clear all steps stop states
1138 stp->more_tokens = 1;
1139 stp->need_new_token = 1;
1140 stp = stp->previous;
1143 // finally convert UTF8 to UTF16 string
1144 icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status);
1146 if (U_FAILURE(*status))
1154 int icu_chain_next_token(struct icu_chain * chain,
1159 if (!chain || !chain->steps)
1162 success = icu_chain_step_next_token(chain, chain->steps, status);
1165 chain->token_count++;
1166 return chain->token_count;
1172 int icu_chain_get_token_count(struct icu_chain * chain)
1177 return chain->token_count;
1182 const char * icu_chain_get_display(struct icu_chain * chain)
1184 if (chain->display8)
1185 return icu_buf_utf8_to_cstr(chain->display8);
1190 const char * icu_chain_get_norm(struct icu_chain * chain)
1193 return icu_buf_utf8_to_cstr(chain->norm8);
1198 const char * icu_chain_get_sort(struct icu_chain * chain)
1201 return icu_buf_utf8_to_cstr(chain->sort8);
1217 * indent-tabs-mode: nil
1219 * vim: shiftwidth=4 tabstop=8 expandtab