Factor relevance charset normalization out to a separate implementation

[pazpar2-moved-to-github.git] / src / icu_I18N.h

diff --git a/src/icu_I18N.h b/src/icu_I18N.h
index eb44204..df6cd2d 100644 (file)
--- a/src/icu_I18N.h
+++ b/src/icu_I18N.h
@@ -1,23 +1,23 @@
-/* $Id: icu_I18N.h,v 1.5 2007-05-07 09:31:36 marc Exp $
+/* $Id: icu_I18N.h,v 1.8 2007-05-09 14:01:21 marc Exp $
    Copyright (c) 2006-2007, Index Data.
 
-This file is part of Pazpar2.
+   This file is part of Pazpar2.
 
-Pazpar2 is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
+   Pazpar2 is free software; you can redistribute it and/or modify it under
+   the terms of the GNU General Public License as published by the Free
+   Software Foundation; either version 2, or (at your option) any later
+   version.
 
-Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
+   Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+   for more details.
 
-You should have received a copy of the GNU General Public License
-along with Pazpar2; see the file LICENSE.  If not, write to the
-Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
-02111-1307, USA.
- */
+   You should have received a copy of the GNU General Public License
+   along with Pazpar2; see the file LICENSE.  If not, write to the
+   Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+   02111-1307, USA.
+*/
 
 #ifndef ICU_I18NL_H
 #define ICU_I18NL_H
@@ -35,10 +35,19 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 //#include <unicode/ucnv.h>     /* C   Converter API    */
 //#include <unicode/ustring.h>  /* some more string fcns*/
 //#include <unicode/uloc.h>
-//#include <unicode/ubrk.h>
+#include <unicode/ubrk.h>
 //#include <unicode/unistr.h>
 
 
+// forward declarations
+//struct UBreakIterator;
+
+
+
+
+// declared structs and functions
+
+
 int icu_check_status (UErrorCode status);
 
 struct icu_buf_utf16
@@ -76,42 +85,58 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
                                     const char * src8cstr,
                                     UErrorCode * status);
 
+
+UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
+                             struct icu_buf_utf16 * src16,
+                             UErrorCode * status);
+
+int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
+                      struct icu_buf_utf16 * src16,
+                      const char *locale, char action,
+                      UErrorCode *status);
+
 UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
                                    struct icu_buf_utf8 * dest8, 
                                    struct icu_buf_utf16 * src16,
                                    UErrorCode * status);
 
-
-
-
-
-
-
-
-
-// CRAP to Follow here ...
-
-#if 0
-struct icu_termmap
+struct icu_tokenizer
 {
-    char * sort_key;   // standard C string '\0' terminated 
-    char * norm_term;  // standard C utf-8 string
-    char * disp_term;  // standard C utf-8 string
+  char locale[16];
+  char action;
+  UBreakIterator* bi;
+  struct icu_buf_utf16 * buf16;
+  int32_t token_count;
+  int32_t token_id;
+  int32_t token_start;
+  int32_t token_end;
+  // keep always invariant
+  // 0 <= token_start 
+  //   <= token_end 
+  //   <= buf16->utf16_len
+  // and invariant
+  // 0 <= token_id <= token_count
 };
 
-struct icu_termmap * icu_termmap_create(NMEM nmem);
+struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
+                                            UErrorCode *status);
+
+void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer);
+
+int icu_tokenizer_attach(struct icu_tokenizer * tokenizer, 
+                         struct icu_buf_utf16 * src16, UErrorCode *status);
 
-int icu_termmap_cmp(const void *vp1, const void *vp2);
+int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer, 
+                                 struct icu_buf_utf16 * tkn16, 
+                                 UErrorCode *status);
 
-char * icu_casemap(NMEM nmem, char *buf, size_t buf_cap, 
-                   size_t *dest8_len,  const char *src8,
-                   const char *locale, char action);
+int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer);
+int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer);
+int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer);
+int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer);
+int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer);
 
-char * icu_sortmap(NMEM nmem, char *buf, size_t buf_cap, 
-                   size_t *dest8_len,  const char *src8,
-                   const char *locale);
 
-#endif // 0
 
 
 #endif // HAVE_ICU