The _relevance_-name was relevant when charset system was only used
to normalize relevance terms, but is inappropriate when it's used to
normalize 4 different types of terms.
static pp2_charset_t pp2_charset_create(struct icu_chain * icu_chn);
static pp2_charset_t pp2_charset_create_a_to_z(void);
static void pp2_charset_destroy(pp2_charset_t pct);
static pp2_charset_t pp2_charset_create(struct icu_chain * icu_chn);
static pp2_charset_t pp2_charset_create_a_to_z(void);
static void pp2_charset_destroy(pp2_charset_t pct);
-static pp2_relevance_token_t pp2_relevance_tokenize(pp2_charset_t pct);
+static pp2_charset_token_t pp2_charset_tokenize(pp2_charset_t pct);
/* charset handle */
struct pp2_charset_s {
/* charset handle */
struct pp2_charset_s {
- const char *(*token_next_handler)(pp2_relevance_token_t prt);
- const char *(*get_sort_handler)(pp2_relevance_token_t prt);
- const char *(*get_display_handler)(pp2_relevance_token_t prt);
+ const char *(*token_next_handler)(pp2_charset_token_t prt);
+ const char *(*get_sort_handler)(pp2_charset_token_t prt);
+ const char *(*get_display_handler)(pp2_charset_token_t prt);
#if YAZ_HAVE_ICU
struct icu_chain * icu_chn;
UErrorCode icu_sts;
#endif
};
#if YAZ_HAVE_ICU
struct icu_chain * icu_chn;
UErrorCode icu_sts;
#endif
};
-static const char *pp2_relevance_token_null(pp2_relevance_token_t prt);
-static const char *pp2_relevance_token_a_to_z(pp2_relevance_token_t prt);
-static const char *pp2_get_sort_ascii(pp2_relevance_token_t prt);
-static const char *pp2_get_display_ascii(pp2_relevance_token_t prt);
+static const char *pp2_charset_token_null(pp2_charset_token_t prt);
+static const char *pp2_charset_token_a_to_z(pp2_charset_token_t prt);
+static const char *pp2_get_sort_ascii(pp2_charset_token_t prt);
+static const char *pp2_get_display_ascii(pp2_charset_token_t prt);
-static const char *pp2_relevance_token_icu(pp2_relevance_token_t prt);
-static const char *pp2_get_sort_icu(pp2_relevance_token_t prt);
-static const char *pp2_get_display_icu(pp2_relevance_token_t prt);
+static const char *pp2_charset_token_icu(pp2_charset_token_t prt);
+static const char *pp2_get_sort_icu(pp2_charset_token_t prt);
+static const char *pp2_get_display_icu(pp2_charset_token_t prt);
#endif
/* tokenzier handle */
#endif
/* tokenzier handle */
-struct pp2_relevance_token_s {
+struct pp2_charset_token_s {
const char *cp; /* unnormalized buffer we're tokenizing */
const char *last_cp; /* pointer to last token we're dealing with */
pp2_charset_t pct; /* our main charset handle (type+config) */
const char *cp; /* unnormalized buffer we're tokenizing */
const char *last_cp; /* pointer to last token we're dealing with */
pp2_charset_t pct; /* our main charset handle (type+config) */
pp2_charset_t pp2_charset_create_a_to_z(void)
{
pp2_charset_t pct = pp2_charset_create(0);
pp2_charset_t pp2_charset_create_a_to_z(void)
{
pp2_charset_t pct = pp2_charset_create(0);
- pct->token_next_handler = pp2_relevance_token_a_to_z;
+ pct->token_next_handler = pp2_charset_token_a_to_z;
{
pp2_charset_t pct = xmalloc(sizeof(*pct));
{
pp2_charset_t pct = xmalloc(sizeof(*pct));
- pct->token_next_handler = pp2_relevance_token_null;
+ pct->token_next_handler = pp2_charset_token_null;
pct->get_sort_handler = pp2_get_sort_ascii;
pct->get_display_handler = pp2_get_display_ascii;
#if YAZ_HAVE_ICU
pct->get_sort_handler = pp2_get_sort_ascii;
pct->get_display_handler = pp2_get_display_ascii;
#if YAZ_HAVE_ICU
{
pct->icu_chn = icu_chn;
pct->icu_sts = U_ZERO_ERROR;
{
pct->icu_chn = icu_chn;
pct->icu_sts = U_ZERO_ERROR;
- pct->token_next_handler = pp2_relevance_token_icu;
+ pct->token_next_handler = pp2_charset_token_icu;
pct->get_sort_handler = pp2_get_sort_icu;
pct->get_display_handler = pp2_get_display_icu;
}
pct->get_sort_handler = pp2_get_sort_icu;
pct->get_display_handler = pp2_get_display_icu;
}
-pp2_relevance_token_t pp2_relevance_create(pp2_charset_fact_t pft,
- const char *id)
+pp2_charset_token_t pp2_charset_token_create(pp2_charset_fact_t pft,
+ const char *id)
{
struct pp2_charset_entry *pce;
for (pce = pft->list; pce; pce = pce->next)
if (!strcmp(id, pce->name))
{
struct pp2_charset_entry *pce;
for (pce = pft->list; pce; pce = pce->next)
if (!strcmp(id, pce->name))
- return pp2_relevance_tokenize(pce->pct);
+ return pp2_charset_tokenize(pce->pct);
-pp2_relevance_token_t pp2_relevance_tokenize(pp2_charset_t pct)
+pp2_charset_token_t pp2_charset_tokenize(pp2_charset_t pct)
- pp2_relevance_token_t prt = xmalloc(sizeof(*prt));
+ pp2_charset_token_t prt = xmalloc(sizeof(*prt));
-void pp2_relevance_first(pp2_relevance_token_t prt,
- const char *buf,
- int skip_article)
+void pp2_charset_token_first(pp2_charset_token_t prt,
+ const char *buf, int skip_article)
-void pp2_relevance_token_destroy(pp2_relevance_token_t prt)
+void pp2_charset_token_destroy(pp2_charset_token_t prt)
{
assert(prt);
#if YAZ_HAVE_ICU
{
assert(prt);
#if YAZ_HAVE_ICU
-const char *pp2_relevance_token_next(pp2_relevance_token_t prt)
+const char *pp2_charset_token_next(pp2_charset_token_t prt)
{
assert(prt);
return (prt->pct->token_next_handler)(prt);
}
{
assert(prt);
return (prt->pct->token_next_handler)(prt);
}
-const char *pp2_get_sort(pp2_relevance_token_t prt)
+const char *pp2_get_sort(pp2_charset_token_t prt)
{
return prt->pct->get_sort_handler(prt);
}
{
return prt->pct->get_sort_handler(prt);
}
-const char *pp2_get_display(pp2_relevance_token_t prt)
+const char *pp2_get_display(pp2_charset_token_t prt)
{
return prt->pct->get_display_handler(prt);
}
{
return prt->pct->get_display_handler(prt);
}
/* original tokenizer with our tokenize interface, but we
add +1 to ensure no '\0' are in our string (except for EOF)
*/
/* original tokenizer with our tokenize interface, but we
add +1 to ensure no '\0' are in our string (except for EOF)
*/
-static const char *pp2_relevance_token_a_to_z(pp2_relevance_token_t prt)
+static const char *pp2_charset_token_a_to_z(pp2_charset_token_t prt)
{
const char *cp = prt->cp;
int c;
{
const char *cp = prt->cp;
int c;
return wrbuf_cstr(prt->norm_str);
}
return wrbuf_cstr(prt->norm_str);
}
-static const char *pp2_get_sort_ascii(pp2_relevance_token_t prt)
+static const char *pp2_get_sort_ascii(pp2_charset_token_t prt)
{
if (prt->last_cp == 0)
return 0;
{
if (prt->last_cp == 0)
return 0;
-static const char *pp2_get_display_ascii(pp2_relevance_token_t prt)
+static const char *pp2_get_display_ascii(pp2_charset_token_t prt)
{
if (prt->last_cp == 0)
return 0;
{
if (prt->last_cp == 0)
return 0;
-static const char *pp2_relevance_token_null(pp2_relevance_token_t prt)
+static const char *pp2_charset_token_null(pp2_charset_token_t prt)
{
const char *cp = prt->cp;
{
const char *cp = prt->cp;
-static const char *pp2_relevance_token_icu(pp2_relevance_token_t prt)
+static const char *pp2_charset_token_icu(pp2_charset_token_t prt)
{
if (icu_iter_next(prt->iter))
{
{
if (icu_iter_next(prt->iter))
{
-static const char *pp2_get_sort_icu(pp2_relevance_token_t prt)
+static const char *pp2_get_sort_icu(pp2_charset_token_t prt)
{
return icu_iter_get_sortkey(prt->iter);
}
{
return icu_iter_get_sortkey(prt->iter);
}
-static const char *pp2_get_display_icu(pp2_relevance_token_t prt)
+static const char *pp2_get_display_icu(pp2_charset_token_t prt)
{
return icu_iter_get_display(prt->iter);
}
{
return icu_iter_get_display(prt->iter);
}
#include <yaz/wrbuf.h>
#include <yaz/xmltypes.h>
#include <yaz/wrbuf.h>
#include <yaz/xmltypes.h>
-typedef struct pp2_relevance_token_s *pp2_relevance_token_t;
+typedef struct pp2_charset_token_s *pp2_charset_token_t;
typedef struct pp2_charset_fact_s *pp2_charset_fact_t;
typedef struct pp2_charset_fact_s *pp2_charset_fact_t;
-void pp2_relevance_first(pp2_relevance_token_t prt,
- const char *buf,
- int skip_article);
-
-void pp2_relevance_token_destroy(pp2_relevance_token_t prt);
-const char *pp2_relevance_token_next(pp2_relevance_token_t prt);
-const char *pp2_get_sort(pp2_relevance_token_t prt);
-const char *pp2_get_display(pp2_relevance_token_t prt);
-
pp2_charset_fact_t pp2_charset_fact_create(void);
void pp2_charset_fact_destroy(pp2_charset_fact_t pft);
int pp2_charset_fact_define(pp2_charset_fact_t pft,
xmlNode *xml_node, const char *default_id);
pp2_charset_fact_t pp2_charset_fact_create(void);
void pp2_charset_fact_destroy(pp2_charset_fact_t pft);
int pp2_charset_fact_define(pp2_charset_fact_t pft,
xmlNode *xml_node, const char *default_id);
-pp2_relevance_token_t pp2_relevance_create(pp2_charset_fact_t pft,
- const char *id);
void pp2_charset_fact_incref(pp2_charset_fact_t pft);
void pp2_charset_fact_incref(pp2_charset_fact_t pft);
+pp2_charset_token_t pp2_charset_token_create(pp2_charset_fact_t pft,
+ const char *id);
+
+void pp2_charset_token_first(pp2_charset_token_t prt,
+ const char *buf,
+ int skip_article);
+void pp2_charset_token_destroy(pp2_charset_token_t prt);
+const char *pp2_charset_token_next(pp2_charset_token_t prt);
+const char *pp2_get_sort(pp2_charset_token_t prt);
+const char *pp2_get_display(pp2_charset_token_t prt);
+
int *doc_frequency_vec;
int vec_len;
struct word_entry *entries;
int *doc_frequency_vec;
int vec_len;
struct word_entry *entries;
- pp2_relevance_token_t prt;
+ pp2_charset_token_t prt;
-static struct word_entry *build_word_entries(pp2_relevance_token_t prt,
+static struct word_entry *build_word_entries(pp2_charset_token_t prt,
NMEM nmem,
const char **terms)
{
NMEM nmem,
const char **terms)
{
- pp2_relevance_first(prt, *p, 0);
- while ((norm_str = pp2_relevance_token_next(prt)))
+ pp2_charset_token_first(prt, *p, 0);
+ while ((norm_str = pp2_charset_token_next(prt)))
add_word_entry(nmem, &entries, norm_str, termno);
termno++;
}
add_word_entry(nmem, &entries, norm_str, termno);
termno++;
}
const char *norm_str;
int i, length = 0;
const char *norm_str;
int i, length = 0;
- pp2_relevance_first(r->prt, words, 0);
+ pp2_charset_token_first(r->prt, words, 0);
for (i = 1; i < r->vec_len; i++)
mult[i] = 0;
for (i = 1; i < r->vec_len; i++)
mult[i] = 0;
- while ((norm_str = pp2_relevance_token_next(r->prt)))
+ while ((norm_str = pp2_charset_token_next(r->prt)))
{
int res = word_entry_match(r->entries, norm_str);
if (res)
{
int res = word_entry_match(r->entries, norm_str);
if (res)
res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int));
memset(res->doc_frequency_vec, 0, res->vec_len * sizeof(int));
res->nmem = nmem;
res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int));
memset(res->doc_frequency_vec, 0, res->vec_len * sizeof(int));
res->nmem = nmem;
- res->prt = pp2_relevance_create(pft, "relevance");
+ res->prt = pp2_charset_token_create(pft, "relevance");
res->entries = build_word_entries(res->prt, nmem, terms);
return res;
}
res->entries = build_word_entries(res->prt, nmem, terms);
return res;
}
- pp2_relevance_token_destroy((*rp)->prt);
+ pp2_charset_token_destroy((*rp)->prt);
void add_facet(struct session *s, const char *type, const char *value, int count)
{
struct conf_service *service = s->service;
void add_facet(struct session *s, const char *type, const char *value, int count)
{
struct conf_service *service = s->service;
- pp2_relevance_token_t prt;
+ pp2_charset_token_t prt;
const char *facet_component;
WRBUF facet_wrbuf = wrbuf_alloc();
WRBUF display_wrbuf = wrbuf_alloc();
const char *facet_component;
WRBUF facet_wrbuf = wrbuf_alloc();
WRBUF display_wrbuf = wrbuf_alloc();
if (!icu_chain_id)
icu_chain_id = "facet";
if (!icu_chain_id)
icu_chain_id = "facet";
- prt = pp2_relevance_create(service->charsets, icu_chain_id);
+ prt = pp2_charset_token_create(service->charsets, icu_chain_id);
if (!prt)
{
yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'",
if (!prt)
{
yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'",
wrbuf_destroy(display_wrbuf);
return;
}
wrbuf_destroy(display_wrbuf);
return;
}
- pp2_relevance_first(prt, value, 0);
- while ((facet_component = pp2_relevance_token_next(prt)))
+ pp2_charset_token_first(prt, value, 0);
+ while ((facet_component = pp2_charset_token_next(prt)))
{
const char *display_component;
if (*facet_component)
{
const char *display_component;
if (*facet_component)
wrbuf_puts(display_wrbuf, display_component);
}
}
wrbuf_puts(display_wrbuf, display_component);
}
}
- pp2_relevance_token_destroy(prt);
+ pp2_charset_token_destroy(prt);
yaz_log(YLOG_LOG, "facet norm=%s", wrbuf_cstr(facet_wrbuf));
yaz_log(YLOG_LOG, "facet display=%s", wrbuf_cstr(display_wrbuf));
yaz_log(YLOG_LOG, "facet norm=%s", wrbuf_cstr(facet_wrbuf));
yaz_log(YLOG_LOG, "facet display=%s", wrbuf_cstr(display_wrbuf));
if (value)
{
const char *norm_str;
if (value)
{
const char *norm_str;
- pp2_relevance_token_t prt =
- pp2_relevance_create(service->charsets, "mergekey");
+ pp2_charset_token_t prt =
+ pp2_charset_token_create(service->charsets, "mergekey");
- pp2_relevance_first(prt, (const char *) value, 0);
+ pp2_charset_token_first(prt, (const char *) value, 0);
if (wrbuf_len(norm_wr) > 0)
wrbuf_puts(norm_wr, " ");
wrbuf_puts(norm_wr, name);
while ((norm_str =
if (wrbuf_len(norm_wr) > 0)
wrbuf_puts(norm_wr, " ");
wrbuf_puts(norm_wr, name);
while ((norm_str =
- pp2_relevance_token_next(prt)))
+ pp2_charset_token_next(prt)))
- pp2_relevance_token_destroy(prt);
+ pp2_charset_token_destroy(prt);
if (mergekey)
{
const char *norm_str;
if (mergekey)
{
const char *norm_str;
- pp2_relevance_token_t prt =
- pp2_relevance_create(service->charsets, "mergekey");
+ pp2_charset_token_t prt =
+ pp2_charset_token_create(service->charsets, "mergekey");
- pp2_relevance_first(prt, (const char *) mergekey, 0);
- while ((norm_str = pp2_relevance_token_next(prt)))
+ pp2_charset_token_first(prt, (const char *) mergekey, 0);
+ while ((norm_str = pp2_charset_token_next(prt)))
wrbuf_puts(norm_wr, norm_str);
}
}
wrbuf_puts(norm_wr, norm_str);
}
}
- pp2_relevance_token_destroy(prt);
+ pp2_charset_token_destroy(prt);
xmlFree(mergekey);
}
else
xmlFree(mergekey);
}
else
// now parsing XML record and adding data to cluster or record metadata
for (n = root->children; n; n = n->next)
{
// now parsing XML record and adding data to cluster or record metadata
for (n = root->children; n; n = n->next)
{
- pp2_relevance_token_t prt;
+ pp2_charset_token_t prt;
if (type)
xmlFree(type);
if (value)
if (type)
xmlFree(type);
if (value)
nmem_malloc(se->nmem,
sizeof(union data_types));
nmem_malloc(se->nmem,
sizeof(union data_types));
- prt = pp2_relevance_create(service->charsets, "sort");
+ prt =
+ pp2_charset_token_create(service->charsets, "sort");
- pp2_relevance_first(prt, rec_md->data.text.disp,
- skip_article);
+ pp2_charset_token_first(prt, rec_md->data.text.disp,
+ skip_article);
- pp2_relevance_token_next(prt);
+ pp2_charset_token_next(prt);
sort_str = pp2_get_sort(prt);
sort_str = pp2_get_sort(prt);
}
cluster->sortkeys[sk_field_id]->text.sort =
nmem_strdup(se->nmem, sort_str);
}
cluster->sortkeys[sk_field_id]->text.sort =
nmem_strdup(se->nmem, sort_str);
- pp2_relevance_token_destroy(prt);
+ pp2_charset_token_destroy(prt);