X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Ficu_tokenizer.c;h=f9b4926c8ae410aacac32ab99fa519b3887bdd10;hb=ded9dadedab82a379e0e52d78a61fe9632ac870c;hp=a79fd6266f6d11dd26359aebc6714ca8bcebc5cf;hpb=3a98cd4b508df847d4aa2b7ed7c1a7213766bad3;p=yaz-moved-to-github.git

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index a79fd62..f9b4926 100644
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -1,5 +1,5 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2009 Index Data
+ * Copyright (C) 1995-2010 Index Data
  * See the file LICENSE for details.
  */
 
@@ -26,6 +26,25 @@
 #include <unicode/ustring.h>  /* some more string fcns*/
 #include <unicode/uchar.h>    /* char names           */
 
+struct icu_tokenizer
+{
+    char action;
+    UBreakIterator* bi;
+    struct icu_buf_utf16 * buf16;
+    int32_t token_count;
+    int32_t token_id;
+    int32_t token_start;
+    int32_t token_end;
+/*
+  keep always invariant
+  0 <= token_start 
+  <= token_end 
+  <= buf16->utf16_len
+  and invariant
+  0 <= token_id <= token_count
+*/
+};
+
 struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
                                            UErrorCode *status)
 {