X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Frelevance.c;h=bc9cb1a1c8c7633c34d92c3a283d3c66125daea3;hb=1a3f3cfb2f6aeb743716924d9081d6dce981b49f;hp=708f2bac231275c5d25610fe45cfa9b2f9d7cdb4;hpb=9e586c9793e3f4846f7307ac3a76537dec1aa43d;p=pazpar2-moved-to-github.git diff --git a/src/relevance.c b/src/relevance.c index 708f2ba..bc9cb1a 100644 --- a/src/relevance.c +++ b/src/relevance.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2011 Index Data + Copyright (C) 2006-2012 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -37,75 +37,58 @@ struct relevance NMEM nmem; }; - struct word_entry { const char *norm_str; int termno; + char *ccl_field; struct word_entry *next; }; -static void add_word_entry(NMEM nmem, - struct word_entry **entries, - const char *norm_str, - int term_no) -{ - struct word_entry *ne = nmem_malloc(nmem, sizeof(*ne)); - ne->norm_str = nmem_strdup(nmem, norm_str); - ne->termno = term_no; - - ne->next = *entries; - *entries = ne; -} - - -int word_entry_match(struct word_entry *entries, const char *norm_str) +static int word_entry_match(struct word_entry *entries, const char *norm_str, + const char *rank, int *mult) { for (; entries; entries = entries->next) { if (!strcmp(norm_str, entries->norm_str)) + { + const char *cp = 0; + int no_read = 0; + sscanf(rank, "%d%n", mult, &no_read); + rank += no_read; + while (*rank == ' ') + rank++; + if (no_read > 0 && (cp = strchr(rank, ' '))) + { + if ((cp - rank) == strlen(entries->ccl_field) && + memcmp(entries->ccl_field, rank, cp - rank) == 0) + *mult = atoi(cp + 1); + } return entries->termno; + } } return 0; } -static struct word_entry *build_word_entries(pp2_charset_token_t prt, - NMEM nmem, - const char **terms) -{ - int termno = 1; /* >0 signals THERE is an entry */ - struct word_entry *entries = 0; - const char **p = terms; - - for (; *p; p++) - { - const char *norm_str; - - pp2_charset_token_first(prt, *p, 0); - while ((norm_str = pp2_charset_token_next(prt))) - add_word_entry(nmem, &entries, norm_str, termno); - termno++; - } - return entries; -} - void relevance_countwords(struct relevance *r, struct record_cluster *cluster, - const char *words, int multiplier, const char *name) + const char *words, const char *rank, + const char *name) { int *mult = cluster->term_frequency_vec_tmp; const char *norm_str; int i, length = 0; - pp2_charset_token_first(r->prt, words, 0); for (i = 1; i < r->vec_len; i++) mult[i] = 0; + assert(rank); while ((norm_str = pp2_charset_token_next(r->prt))) { - int res = word_entry_match(r->entries, norm_str); + int local_mult = 0; + int res = word_entry_match(r->entries, norm_str, rank, &local_mult); if (res) { assert(res < r->vec_len); - mult[res] += multiplier; + mult[res] += local_mult; } length++; } @@ -120,21 +103,66 @@ void relevance_countwords(struct relevance *r, struct record_cluster *cluster, cluster->term_frequency_vec[0] += length; } -struct relevance *relevance_create(pp2_charset_fact_t pft, - NMEM nmem, const char **terms) +static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) { - struct relevance *res = nmem_malloc(nmem, sizeof(struct relevance)); - const char **p; + char **words; + int numwords; + char *ccl_field; + int i; + + switch (n->kind) + { + case CCL_RPN_AND: + case CCL_RPN_OR: + case CCL_RPN_NOT: + case CCL_RPN_PROX: + pull_terms(res, n->u.p[0]); + pull_terms(res, n->u.p[1]); + break; + case CCL_RPN_TERM: + nmem_strsplit(res->nmem, " ", n->u.t.term, &words, &numwords); + for (i = 0; i < numwords; i++) + { + const char *norm_str; + + ccl_field = nmem_strdup_null(res->nmem, n->u.t.qual); + + pp2_charset_token_first(res->prt, words[i], 0); + while ((norm_str = pp2_charset_token_next(res->prt))) + { + struct word_entry **e = &res->entries; + while (*e) + e = &(*e)->next; + *e = nmem_malloc(res->nmem, sizeof(**e)); + (*e)->norm_str = nmem_strdup(res->nmem, norm_str); + (*e)->ccl_field = ccl_field; + (*e)->termno = res->vec_len++; + (*e)->next = 0; + } + } + break; + default: + break; + } +} + +struct relevance *relevance_create_ccl(pp2_charset_fact_t pft, + struct ccl_rpn_node *query) +{ + NMEM nmem = nmem_create(); + struct relevance *res = nmem_malloc(nmem, sizeof(*res)); int i; - for (p = terms, i = 0; *p; p++, i++) - ; - res->vec_len = ++i; - res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int)); - memset(res->doc_frequency_vec, 0, res->vec_len * sizeof(int)); res->nmem = nmem; + res->entries = 0; + res->vec_len = 1; res->prt = pp2_charset_token_create(pft, "relevance"); - res->entries = build_word_entries(res->prt, nmem, terms); + + pull_terms(res, query); + + res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int)); + for (i = 0; i < res->vec_len; i++) + res->doc_frequency_vec[i] = 0; return res; } @@ -143,6 +171,7 @@ void relevance_destroy(struct relevance **rp) if (*rp) { pp2_charset_token_destroy((*rp)->prt); + nmem_destroy((*rp)->nmem); *rp = 0; } } @@ -174,7 +203,6 @@ void relevance_newrec(struct relevance *r, struct record_cluster *rec) } } - void relevance_donerecord(struct relevance *r, struct record_cluster *cluster) { int i; @@ -212,7 +240,6 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) } } // Calculate relevance for each document - while (1) { int t; @@ -222,18 +249,7 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) break; for (t = 1; t < rel->vec_len; t++) { - float termfreq; -#if 1 - termfreq = (float) rec->term_frequency_vecf[t]; -#else - if (rec->term_frequency_vec[0]) - { - termfreq = (float) - rec->term_frequency_vec[t] / rec->term_frequency_vec[0] ; - } - else - termfreq = 0.0; -#endif + float termfreq = (float) rec->term_frequency_vecf[t]; relevance += 100000 * (termfreq * idfvec[t] + 0.0000005); } rec->relevance_score = relevance;