tokenizer->action = action;
tokenizer->bi = 0;
- tokenizer->buf16 = 0;
+ tokenizer->buf16 = icu_buf_utf16_create(0);
tokenizer->token_count = 0;
tokenizer->token_id = 0;
tokenizer->token_start = 0;
{
if (tokenizer)
{
+ icu_buf_utf16_destroy(tokenizer->buf16);
if (tokenizer->bi)
ubrk_close(tokenizer->bi);
xfree(tokenizer);
if (!tokenizer || !tokenizer->bi || !src16)
return 0;
- tokenizer->buf16 = src16;
+ icu_buf_utf16_copy(tokenizer->buf16, src16);
+
tokenizer->token_count = 0;
tokenizer->token_id = 0;
tokenizer->token_start = 0;
tokenizer->token_end = 0;
- ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
+ ubrk_setText(tokenizer->bi,
+ tokenizer->buf16->utf16, tokenizer->buf16->utf16_len, status);
if (U_FAILURE(*status))
return 0;