-static int icu_chain_step_next_token(struct icu_chain * chain,
- struct icu_chain_step * step,
- UErrorCode *status)
-{
- struct icu_buf_utf16 * src16 = 0;
- int got_new_token = 0;
-
- if (!chain || !chain->src16 || !step || !step->more_tokens)
- return 0;
-
- /* assign utf16 src buffers as needed, advance in previous steps
- tokens until non-zero token met, and setting stop condition */
-
- if (step->previous)
- {
- src16 = step->previous->buf16;
- /* tokens might be killed in previous steps, therefore looping */
-
- while (step->need_new_token
- && step->previous->more_tokens
- && !got_new_token)
- got_new_token
- = icu_chain_step_next_token(chain, step->previous, status);
- }
- else
- { /* first step can only work once on chain->src16 input buffer */
- src16 = chain->src16;
- step->more_tokens = 0;
- got_new_token = 1;
- }
-
- if (!src16)
- return 0;
-
- /* stop if nothing to process */
- if (step->need_new_token && !got_new_token)
- {
- step->more_tokens = 0;
- return 0;
- }
-
- /* either an old token not finished yet, or a new token, thus
- perform the work, eventually put this steps output in
- step->buf16 or the chains UTF8 output buffers */
-
- switch (step->type)
- {
- case ICU_chain_step_type_display:
- icu_utf16_to_utf8(chain->display8, src16, status);
- break;
- case ICU_chain_step_type_casemap:
- icu_casemap_casemap(step->u.casemap,
- step->buf16, src16, status,
- chain->locale);
- break;
- case ICU_chain_step_type_transform:
- case ICU_chain_step_type_transliterate:
- icu_transform_trans(step->u.transform,
- step->buf16, src16, status);
- break;
- case ICU_chain_step_type_tokenize:
- /* attach to new src16 token only first time during splitting */
- if (step->need_new_token)
- {
- icu_tokenizer_attach(step->u.tokenizer, src16, status);
- step->need_new_token = 0;
- }
-
- /* splitting one src16 token into multiple buf16 tokens */
- step->more_tokens
- = icu_tokenizer_next_token(step->u.tokenizer,
- step->buf16, status);
-
- /* make sure to get new previous token if this one had been used up
- by recursive call to _same_ step */
-
- if (!step->more_tokens)
- {
- step->more_tokens = icu_chain_step_next_token(chain, step, status);
- return step->more_tokens; /* avoid one token count too much! */
- }
- break;
- default:
- return 0;
- break;
- }
-
- if (U_FAILURE(*status))
- return 0;
-
- /* if token disappered into thin air, tell caller */
- /* if (!step->buf16->utf16_len && !step->more_tokens) */
- /* return 0; */
-
- return 1;
-}
-