-
- //if (0 == step->more_tokens)
- //return 0;
- break;
- default:
- return 0;
- break;
- }
-
- if (U_FAILURE(*status))
- return 0;
-
- /* if token disappered into thin air, tell caller */
- if (!step->buf16->utf16_len)
- return 0;
-
- if (U_FAILURE(*status))
- return 0;
-
- return 1;
-}
-
-
-#if 0 /* backup */
-int icu_chain_step_next_token_BAK(struct icu_chain * chain,
- struct icu_chain_step * step,
- UErrorCode *status)
-{
- struct icu_buf_utf16 * src16 = 0;
-
- if (!chain || !chain->src16 || !step || !step->more_tokens)
- return 0;
-
- /* assign utf16 src buffers as neeed, advance in previous steps
- tokens until non-zero token met, and setting stop condition
- */
- if (step->previous){
- src16 = step->previous->buf16;
- if (step->need_new_token)
- step->more_tokens
- = icu_chain_step_next_token(chain, step->previous, status);
- }
- else { /* first step can only work once on chain->src16 input buffer */
- src16 = chain->src16;
- step->more_tokens = 1;
- }
-
- /* stop if nothing to process
- i.e new token source was not properly assigned
- or I did not get any tokens from previous step
- */
- if (!step->more_tokens || !src16)
- return 0;
-
- /* perform the work, eventually put this steps output in
- step->buf16 or the chains UTF8 output buffers */
- switch(step->type) {
- case ICU_chain_step_type_display:
- icu_utf16_to_utf8(chain->display8, src16, status);
- break;
- case ICU_chain_step_type_index:
- icu_utf16_to_utf8(chain->norm8, src16, status);
- break;
- case ICU_chain_step_type_sortkey:
- icu_utf16_to_utf8(chain->sort8, src16, status);
- break;
- case ICU_chain_step_type_casemap:
- icu_casemap_casemap(step->u.casemap,
- step->buf16, src16, status);
- break;
- case ICU_chain_step_type_normalize:
- icu_normalizer_normalize(step->u.normalizer,
- step->buf16, src16, status);
- break;
- case ICU_chain_step_type_tokenize:
- /* attach to new src16 token only first time during splitting */
- if (step->need_new_token){
- icu_tokenizer_attach(step->u.tokenizer, src16, status);
- step->need_new_token = 0;
- }
- /* splitting one src16 token into multiple buf16 tokens */
- step->more_tokens
- = icu_tokenizer_next_token(step->u.tokenizer,
- step->buf16, status);
- /* make sure to get new previous token if this one had been used up */
- if (step->previous && !step->more_tokens){
- if (icu_chain_step_next_token(chain, step->previous, status)){
- icu_tokenizer_attach(step->u.tokenizer, src16, status);
- step->need_new_token = 0;
- step->more_tokens
- = icu_tokenizer_next_token(step->u.tokenizer,
- step->buf16, status);
- }