1 /* $Id: icu_I18N.c,v 1.17 2007-05-20 19:00:17 marc Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
28 #include <yaz/timing.h>
41 #include <unicode/ustring.h> /* some more string fcns*/
42 #include <unicode/uchar.h> /* char names */
45 //#include <unicode/ustdio.h>
46 //#include <unicode/utypes.h> /* Basic ICU data types */
47 #include <unicode/ucol.h>
48 //#include <unicode/ucnv.h> /* C Converter API */
49 //#include <unicode/uloc.h>
50 //#include <unicode/ubrk.h>
51 /* #include <unicode/unistr.h> */
56 int icu_check_status (UErrorCode status)
58 if(U_FAILURE(status)){
60 "ICU: %d %s\n", status, u_errorName(status));
69 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
71 struct icu_buf_utf16 * buf16
72 = (struct icu_buf_utf16 *) malloc(sizeof(struct icu_buf_utf16));
79 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
80 buf16->utf16[0] = (UChar) 0;
81 buf16->utf16_cap = capacity;
87 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
92 if (0 == buf16->utf16)
93 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
96 = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity);
97 buf16->utf16[0] = (UChar) 0;
99 buf16->utf16_cap = capacity;
105 buf16->utf16_len = 0;
106 buf16->utf16_cap = 0;
114 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
115 struct icu_buf_utf16 * src16)
121 if (dest16->utf16_cap < src16->utf16_len)
122 icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
124 u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
125 dest16->utf16_len = src16->utf16_len;
131 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
145 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
147 struct icu_buf_utf8 * buf8
148 = (struct icu_buf_utf8 *) malloc(sizeof(struct icu_buf_utf8));
155 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
156 buf8->utf8[0] = (uint8_t) 0;
157 buf8->utf8_cap = capacity;
164 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
170 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
173 = (uint8_t *) realloc(buf8->utf8,
174 sizeof(uint8_t) * capacity);
175 buf8->utf8[0] = (uint8_t) 0;
177 buf8->utf8_cap = capacity;
192 struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8,
193 struct icu_buf_utf8 * src8)
200 if (dest8->utf8_cap < src8->utf8_len)
201 icu_buf_utf8_resize(dest8, src8->utf8_len * 2);
203 strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len);
210 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
221 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
222 struct icu_buf_utf8 * src8,
225 int32_t utf16_len = 0;
227 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
229 (const char *) src8->utf8, src8->utf8_len, status);
231 // check for buffer overflow, resize and retry
232 if (*status == U_BUFFER_OVERFLOW_ERROR
233 //|| dest16->utf16_len > dest16->utf16_cap
235 icu_buf_utf16_resize(dest16, utf16_len * 2);
236 *status = U_ZERO_ERROR;
237 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
239 (const char *) src8->utf8, src8->utf8_len, status);
242 //if (*status != U_BUFFER_OVERFLOW_ERROR
243 if (U_SUCCESS(*status)
244 && utf16_len < dest16->utf16_cap)
245 dest16->utf16_len = utf16_len;
247 dest16->utf16[0] = (UChar) 0;
248 dest16->utf16_len = 0;
256 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
257 const char * src8cstr,
260 size_t src8cstr_len = 0;
261 int32_t utf16_len = 0;
263 src8cstr_len = strlen(src8cstr);
265 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
267 src8cstr, src8cstr_len, status);
269 // check for buffer overflow, resize and retry
270 if (*status == U_BUFFER_OVERFLOW_ERROR
271 //|| dest16->utf16_len > dest16->utf16_cap
273 icu_buf_utf16_resize(dest16, utf16_len * 2);
274 *status = U_ZERO_ERROR;
275 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
277 src8cstr, src8cstr_len, status);
280 // if (*status != U_BUFFER_OVERFLOW_ERROR
281 if (U_SUCCESS(*status)
282 && utf16_len < dest16->utf16_cap)
283 dest16->utf16_len = utf16_len;
285 dest16->utf16[0] = (UChar) 0;
286 dest16->utf16_len = 0;
295 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
296 struct icu_buf_utf16 * src16,
299 int32_t utf8_len = 0;
301 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
303 src16->utf16, src16->utf16_len, status);
305 // check for buffer overflow, resize and retry
306 if (*status == U_BUFFER_OVERFLOW_ERROR
307 //|| dest8->utf8_len > dest8->utf8_cap
309 icu_buf_utf8_resize(dest8, utf8_len * 2);
310 *status = U_ZERO_ERROR;
311 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
313 src16->utf16, src16->utf16_len, status);
317 //if (*status != U_BUFFER_OVERFLOW_ERROR
318 if (U_SUCCESS(*status)
319 && utf8_len < dest8->utf8_cap)
320 dest8->utf8_len = utf8_len;
322 dest8->utf8[0] = (uint8_t) 0;
331 struct icu_casemap * icu_casemap_create(const char *locale, char action,
334 struct icu_casemap * casemap
335 = (struct icu_casemap *) malloc(sizeof(struct icu_casemap));
336 strcpy(casemap->locale, locale);
337 casemap->action = action;
339 switch(casemap->action) {
349 icu_casemap_destroy(casemap);
356 void icu_casemap_destroy(struct icu_casemap * casemap)
363 int icu_casemap_casemap(struct icu_casemap * casemap,
364 struct icu_buf_utf16 * dest16,
365 struct icu_buf_utf16 * src16,
371 return icu_utf16_casemap(dest16, src16,
372 casemap->locale, casemap->action, status);
376 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
377 struct icu_buf_utf16 * src16,
378 const char *locale, char action,
381 int32_t dest16_len = 0;
385 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
386 src16->utf16, src16->utf16_len,
390 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
391 src16->utf16, src16->utf16_len,
395 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
396 src16->utf16, src16->utf16_len,
400 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
401 src16->utf16, src16->utf16_len,
402 U_FOLD_CASE_DEFAULT, status);
406 return U_UNSUPPORTED_ERROR;
410 // check for buffer overflow, resize and retry
411 if (*status == U_BUFFER_OVERFLOW_ERROR
412 && dest16 != src16 // do not resize if in-place conversion
413 //|| dest16_len > dest16->utf16_cap
415 icu_buf_utf16_resize(dest16, dest16_len * 2);
416 *status = U_ZERO_ERROR;
421 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
422 src16->utf16, src16->utf16_len,
426 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
427 src16->utf16, src16->utf16_len,
431 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
432 src16->utf16, src16->utf16_len,
436 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
437 src16->utf16, src16->utf16_len,
438 U_FOLD_CASE_DEFAULT, status);
442 return U_UNSUPPORTED_ERROR;
447 if (U_SUCCESS(*status)
448 && dest16_len < dest16->utf16_cap)
449 dest16->utf16_len = dest16_len;
451 dest16->utf16[0] = (UChar) 0;
452 dest16->utf16_len = 0;
460 UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
461 struct icu_buf_utf8 * dest8,
462 struct icu_buf_utf16 * src16,
466 int32_t sortkey_len = 0;
468 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
469 dest8->utf8, dest8->utf8_cap);
471 // check for buffer overflow, resize and retry
472 if (sortkey_len > dest8->utf8_cap) {
473 icu_buf_utf8_resize(dest8, sortkey_len * 2);
474 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
475 dest8->utf8, dest8->utf8_cap);
478 if (U_SUCCESS(*status)
480 dest8->utf8_len = sortkey_len;
482 dest8->utf8[0] = (UChar) 0;
491 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
494 struct icu_tokenizer * tokenizer
495 = (struct icu_tokenizer *) malloc(sizeof(struct icu_tokenizer));
497 strcpy(tokenizer->locale, locale);
498 tokenizer->action = action;
500 tokenizer->buf16 = 0;
501 tokenizer->token_count = 0;
502 tokenizer->token_id = 0;
503 tokenizer->token_start = 0;
504 tokenizer->token_end = 0;
507 switch(tokenizer->action) {
510 = ubrk_open(UBRK_LINE, tokenizer->locale,
515 = ubrk_open(UBRK_SENTENCE, tokenizer->locale,
520 = ubrk_open(UBRK_WORD, tokenizer->locale,
525 = ubrk_open(UBRK_CHARACTER, tokenizer->locale,
530 = ubrk_open(UBRK_TITLE, tokenizer->locale,
534 *status = U_UNSUPPORTED_ERROR;
539 // ICU error stuff is a very funny business
540 if (U_SUCCESS(*status))
544 icu_tokenizer_destroy(tokenizer);
548 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
552 ubrk_close(tokenizer->bi);
557 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
558 struct icu_buf_utf16 * src16,
561 if (!tokenizer || !tokenizer->bi || !src16)
565 tokenizer->buf16 = src16;
566 tokenizer->token_count = 0;
567 tokenizer->token_id = 0;
568 tokenizer->token_start = 0;
569 tokenizer->token_end = 0;
571 ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
574 if (U_FAILURE(*status))
580 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
581 struct icu_buf_utf16 * tkn16,
584 int32_t tkn_start = 0;
589 if (!tokenizer || !tokenizer->bi
590 || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
593 // never change tokenizer->buf16 and keep always invariant
594 // 0 <= tokenizer->token_start
595 // <= tokenizer->token_end
596 // <= tokenizer->buf16->utf16_len
597 // returns length of token
599 if (0 == tokenizer->token_end) // first call
600 tkn_start = ubrk_first(tokenizer->bi);
601 else //successive calls
602 tkn_start = tokenizer->token_end;
605 tkn_end = ubrk_next(tokenizer->bi);
607 // repairing invariant at end of ubrk, which is UBRK_DONE = -1
608 if (UBRK_DONE == tkn_end)
609 tkn_end = tokenizer->buf16->utf16_len;
611 // copy out if everything is well
612 if(U_FAILURE(*status))
615 // everything OK, now update internal state
616 tkn_len = tkn_end - tkn_start;
619 tokenizer->token_count++;
620 tokenizer->token_id++;
622 tokenizer->token_id = 0;
624 tokenizer->token_start = tkn_start;
625 tokenizer->token_end = tkn_end;
628 // copying into token buffer if it exists
630 if (tkn16->utf16_cap < tkn_len)
631 icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
633 u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
636 tkn16->utf16_len = tkn_len;
643 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
645 return tokenizer->token_id;
648 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
650 return tokenizer->token_start;
653 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
655 return tokenizer->token_end;
658 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
660 return (tokenizer->token_end - tokenizer->token_start);
663 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
665 return tokenizer->token_count;
670 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
674 struct icu_normalizer * normalizer
675 = (struct icu_normalizer *) malloc(sizeof(struct icu_normalizer));
677 normalizer->action = action;
678 normalizer->trans = 0;
679 normalizer->rules16 = icu_buf_utf16_create(0);
680 icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
682 switch(normalizer->action) {
685 = utrans_openU(normalizer->rules16->utf16,
686 normalizer->rules16->utf16_len,
689 normalizer->parse_error, status);
693 = utrans_openU(normalizer->rules16->utf16,
694 normalizer->rules16->utf16_len,
697 normalizer->parse_error, status);
700 *status = U_UNSUPPORTED_ERROR;
705 if (U_SUCCESS(*status))
709 icu_normalizer_destroy(normalizer);
714 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
716 if (normalizer->rules16)
717 icu_buf_utf16_destroy(normalizer->rules16);
718 if (normalizer->trans)
719 utrans_close(normalizer->trans);
726 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
727 struct icu_buf_utf16 * dest16,
728 struct icu_buf_utf16 * src16,
731 if (!normalizer || !normalizer->trans || !src16 || !dest16)
734 if (!icu_buf_utf16_copy(dest16, src16))
737 utrans_transUChars (normalizer->trans,
738 dest16->utf16, &(dest16->utf16_len),
740 0, &(src16->utf16_len), status);
742 if (U_FAILURE(*status)){
743 dest16->utf16[0] = (UChar) 0;
744 dest16->utf16_len = 0;
747 return dest16->utf16_len;
753 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
754 enum icu_chain_step_type type,
755 const uint8_t * rule,
756 struct icu_buf_utf16 * buf16,
759 struct icu_chain_step * step = 0;
761 if(!chain || !type || !rule)
764 step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step));
767 step->more_tokens = 0;
768 step->need_new_token = 1;
775 // create auxilary objects
777 case ICU_chain_step_type_display:
779 case ICU_chain_step_type_norm:
781 case ICU_chain_step_type_sort:
783 case ICU_chain_step_type_casemap:
784 step->u.casemap = icu_casemap_create((char *) chain->locale,
785 (char) rule[0], status);
787 case ICU_chain_step_type_normalize:
788 step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
790 case ICU_chain_step_type_tokenize:
791 step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
792 (char) rule[0], status);
802 void icu_chain_step_destroy(struct icu_chain_step * step){
807 icu_chain_step_destroy(step->previous);
810 case ICU_chain_step_type_display:
812 case ICU_chain_step_type_norm:
814 case ICU_chain_step_type_sort:
816 case ICU_chain_step_type_casemap:
817 icu_casemap_destroy(step->u.casemap);
818 icu_buf_utf16_destroy(step->buf16);
820 case ICU_chain_step_type_normalize:
821 icu_normalizer_destroy(step->u.normalizer);
822 icu_buf_utf16_destroy(step->buf16);
824 case ICU_chain_step_type_tokenize:
825 icu_tokenizer_destroy(step->u.tokenizer);
826 icu_buf_utf16_destroy(step->buf16);
837 struct icu_chain * icu_chain_create(const uint8_t * identifier,
838 const uint8_t * locale)
841 struct icu_chain * chain
842 = (struct icu_chain *) malloc(sizeof(struct icu_chain));
844 strncpy((char *) chain->identifier, (const char *) identifier, 128);
845 chain->identifier[128 - 1] = '\0';
846 strncpy((char *) chain->locale, (const char *) locale, 16);
847 chain->locale[16 - 1] = '\0';
849 chain->token_count = 0;
851 chain->display8 = icu_buf_utf8_create(0);
852 chain->norm8 = icu_buf_utf8_create(0);
853 chain->sort8 = icu_buf_utf8_create(0);
855 chain->src16 = icu_buf_utf16_create(0);
863 void icu_chain_destroy(struct icu_chain * chain)
865 icu_buf_utf8_destroy(chain->display8);
866 icu_buf_utf8_destroy(chain->norm8);
867 icu_buf_utf8_destroy(chain->sort8);
869 icu_buf_utf16_destroy(chain->src16);
871 icu_chain_step_destroy(chain->steps);
875 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
876 enum icu_chain_step_type type,
877 const uint8_t * rule,
880 struct icu_chain_step * step = 0;
881 struct icu_buf_utf16 * src16 = 0;
882 struct icu_buf_utf16 * buf16 = 0;
884 if (!chain || !type || !rule)
887 // assign utf16 src buffers as needed
888 if (chain->steps && chain->steps->buf16)
889 src16 = chain->steps->buf16;
890 else if (chain->src16)
891 src16 = chain->src16;
896 // create utf16 destination buffers as needed, or
898 case ICU_chain_step_type_display:
901 case ICU_chain_step_type_norm:
904 case ICU_chain_step_type_sort:
907 case ICU_chain_step_type_casemap:
908 buf16 = icu_buf_utf16_create(0);
910 case ICU_chain_step_type_normalize:
911 buf16 = icu_buf_utf16_create(0);
913 case ICU_chain_step_type_tokenize:
914 buf16 = icu_buf_utf16_create(0);
920 // create actual chain step with this buffer
921 step = icu_chain_step_create(chain, type, rule, buf16, status);
923 step->previous = chain->steps;
930 int icu_chain_step_next_token(struct icu_chain * chain,
931 struct icu_chain_step * step,
934 struct icu_buf_utf16 * src16 = 0;
936 //printf("icu_chain_step_next_token %d\n", (int) step);
938 if (!chain || !chain->src16 || !step || !step->more_tokens)
941 // assign utf16 src buffers as neeed, advance in previous steps
942 // tokens until non-zero token met, and setting stop condition
944 src16 = step->previous->buf16;
945 if (step->need_new_token)
946 //while (step->more_tokens && !src16->utf16_len)
948 = icu_chain_step_next_token(chain, step->previous, status);
950 else { // first step can only work once on chain->src16 input buffer
951 src16 = chain->src16;
952 step->more_tokens = 1;
955 // stop if nothing to process
956 // i.e new token source was not properly assigned
957 if (!step->more_tokens || !src16) // || !src16->utf16_len
960 //printf("icu_chain_step_next_token %d working\n", (int) step);
963 // perform the work, eventually put this steps output in
964 // step->buf16 or the chains UTF8 output buffers
966 case ICU_chain_step_type_display:
967 icu_utf16_to_utf8(chain->display8, src16, status);
969 case ICU_chain_step_type_norm:
970 icu_utf16_to_utf8(chain->norm8, src16, status);
972 case ICU_chain_step_type_sort:
973 icu_utf16_to_utf8(chain->sort8, src16, status);
975 case ICU_chain_step_type_casemap:
976 icu_casemap_casemap(step->u.casemap,
977 step->buf16, src16, status);
979 case ICU_chain_step_type_normalize:
980 icu_normalizer_normalize(step->u.normalizer,
981 step->buf16, src16, status);
983 case ICU_chain_step_type_tokenize:
984 // attach to new src16 token only first time during splitting
985 if (step->need_new_token){
986 icu_tokenizer_attach(step->u.tokenizer, src16, status);
987 step->need_new_token = 0;
989 // splitting one src16 token into multiple buf16 tokens
991 = icu_tokenizer_next_token(step->u.tokenizer,
992 step->buf16, status);
993 // make sure to get new previous token if this one had been used up
994 if (step->previous && !step->more_tokens){
995 if (icu_chain_step_next_token(chain, step->previous, status)){
996 icu_tokenizer_attach(step->u.tokenizer, src16, status);
997 step->need_new_token = 0;
999 = icu_tokenizer_next_token(step->u.tokenizer,
1000 step->buf16, status);
1003 if (0 == step->more_tokens)
1013 // stop further token processing if last step and
1014 // new tokens are needed from previous (non-existing) step
1015 if (!step->previous && step->need_new_token)
1016 step->more_tokens = 0;
1018 //printf("%d %d %d\n",
1019 // step->more_tokens, src16->utf16_len, step->buf16->utf16_len);
1022 if (U_FAILURE(*status))
1030 int icu_chain_assign_cstr(struct icu_chain * chain,
1031 const char * src8cstr,
1034 struct icu_chain_step * stp = chain->steps;
1036 if (!chain || !src8cstr)
1039 // clear token count
1040 chain->token_count = 0;
1042 // clear all steps stop states
1045 stp->more_tokens = 1;
1046 stp = stp->previous;
1049 // finally convert UTF8 to UTF16 string
1050 icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status);
1052 if (U_FAILURE(*status))
1060 int icu_chain_next_token(struct icu_chain * chain,
1065 if (!chain || !chain->steps)
1068 success = icu_chain_step_next_token(chain, chain->steps, status);
1071 chain->token_count++;
1072 return chain->token_count;
1078 int icu_chain_get_token_count(struct icu_chain * chain)
1083 return chain->token_count;
1088 const char * icu_chain_get_display(struct icu_chain * chain)
1090 if (chain->display8)
1091 return (const char *) chain->display8->utf8;
1096 const char * icu_chain_get_norm(struct icu_chain * chain)
1099 return (const char *) chain->norm8->utf8;
1104 const char * icu_chain_get_sort(struct icu_chain * chain)
1107 return (const char *) chain->sort8->utf8;
1123 * indent-tabs-mode: nil
1125 * vim: shiftwidth=4 tabstop=8 expandtab