9 #include <yaz/yconfig.h>
11 #include <yaz/stemmer.h>
13 #include <yaz/xmalloc.h>
15 #include <libstemmer.h>
17 #include <unicode/ustring.h> /* some more string fcns*/
18 #include <unicode/uchar.h> /* char names */
20 enum stemmer_implementation {
27 // Required for cloning.
31 struct sb_stemmer *sb_stemmer;
35 const char* yaz_stemmer_lookup_charenc(const char *charenc, const char *rule) {
39 const char* yaz_stemmer_lookup_algorithm(const char *locale, const char *rule) {
43 yaz_stemmer_p yaz_stemmer_snowball_create(const char *locale, const char *rule, UErrorCode *status) {
44 const char *charenc = yaz_stemmer_lookup_charenc(locale, rule);
45 const char *algorithm = yaz_stemmer_lookup_algorithm(locale,rule);
46 struct sb_stemmer *stemmer = sb_stemmer_new(algorithm, charenc);
47 yaz_stemmer_p yaz_stemmer;
49 *status = U_ILLEGAL_ARGUMENT_ERROR;
50 yaz_log(YLOG_FATAL, "yaz_stemmer: Failed to create snowball stemmer from locale %srule %s. Showball: charenc %s algorithm %s ",
51 locale, rule, charenc, algorithm);
54 yaz_log(YLOG_DEBUG, "created snowball stemmer: algoritm %s charenc %s ", algorithm, charenc);
55 yaz_stemmer = xmalloc(sizeof(*yaz_stemmer));
56 yaz_stemmer->implementation = yaz_snowball;
58 yaz_stemmer->locale = xstrdup(locale);
59 yaz_stemmer->rule = xstrdup(rule);
60 yaz_stemmer->sb_stemmer = stemmer;
61 yaz_log(YLOG_DEBUG, "created snowball stemmer: algoritm %s charenc %s ", algorithm, charenc);
65 yaz_stemmer_p yaz_stemmer_create(const char *locale, const char *rule, UErrorCode *status) {
66 *status = U_ZERO_ERROR;
67 // dispatch logic required if more algorithms is implemented.
68 yaz_log(YLOG_DEBUG, "create stemmer: locale %s rule %s ", locale, rule);
69 return yaz_stemmer_snowball_create(locale, rule, status);
72 yaz_stemmer_p yaz_stemmer_clone(yaz_stemmer_p stemmer) {
73 UErrorCode error = U_ZERO_ERROR;
76 return yaz_stemmer_create(stemmer->locale, stemmer->rule, &error);
79 void yaz_stemmer_stem(yaz_stemmer_p stemmer, struct icu_buf_utf16 *dst, struct icu_buf_utf16* src, UErrorCode *status)
81 switch(stemmer->implementation) {
83 struct icu_buf_utf8 *utf8_buf = icu_buf_utf8_create(0);
84 icu_utf16_to_utf8(utf8_buf, src, status);
85 if (*status == U_ZERO_ERROR) {
86 const sb_symbol *cstr = (const sb_symbol*) icu_buf_utf8_to_cstr(utf8_buf);
87 const sb_symbol *sb_symbol = sb_stemmer_stem(stemmer->sb_stemmer, cstr, utf8_buf->utf8_len);
89 icu_buf_utf16_copy(dst, src);
92 const char *cstr = (const char *) sb_symbol;
93 icu_utf16_from_utf8_cstr(dst, cstr , status);
96 icu_buf_utf8_destroy(utf8_buf);
100 case yaz_no_operation:
101 yaz_log(YLOG_DEBUG, "Stemmer (No operation) called");
103 // Default return the same as given.
104 icu_buf_utf16_copy(dst, src);
109 void yaz_stemmer_destroy(yaz_stemmer_p stemmer)
111 /* Handle no stemmer correctly */
115 switch (stemmer->implementation) {
117 sb_stemmer_delete(stemmer->sb_stemmer);
120 xfree(stemmer->locale);
121 xfree(stemmer->rule);
125 #endif /* YAZ_HAVE_ICU */