X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;ds=sidebyside;f=src%2Frelevance.c;h=3755b0902c0cff4163bd80d170c72328542ed34d;hb=ee40230fd0e997212b707fafcdec0da154fd1b21;hp=418dc9a0b86ce0256017d63df26adb6c91f5ff5d;hpb=eb673212c641c36afba4cd502e283247de52a015;p=pazpar2-moved-to-github.git diff --git a/src/relevance.c b/src/relevance.c index 418dc9a..3755b09 100644 --- a/src/relevance.c +++ b/src/relevance.c @@ -34,53 +34,80 @@ struct relevance int vec_len; struct word_entry *entries; pp2_charset_token_t prt; + int rank_cluster; NMEM nmem; }; struct word_entry { const char *norm_str; + const char *display_str; int termno; char *ccl_field; struct word_entry *next; }; -int word_entry_match(struct word_entry *entries, const char *norm_str) +static int word_entry_match(struct word_entry *entries, const char *norm_str, + const char *rank, int *mult) { for (; entries; entries = entries->next) { - if (!strcmp(norm_str, entries->norm_str)) + if (*norm_str && !strcmp(norm_str, entries->norm_str)) + { + const char *cp = 0; + int no_read = 0; + sscanf(rank, "%d%n", mult, &no_read); + rank += no_read; + while (*rank == ' ') + rank++; + if (no_read > 0 && (cp = strchr(rank, ' '))) + { + if ((cp - rank) == strlen(entries->ccl_field) && + memcmp(entries->ccl_field, rank, cp - rank) == 0) + *mult = atoi(cp + 1); + } return entries->termno; + } } return 0; } void relevance_countwords(struct relevance *r, struct record_cluster *cluster, - const char *words, int multiplier, const char *name) + const char *words, const char *rank, + const char *name) { int *mult = cluster->term_frequency_vec_tmp; const char *norm_str; int i, length = 0; + struct word_entry *e = r->entries; + WRBUF w = cluster->relevance_explain1; pp2_charset_token_first(r->prt, words, 0); for (i = 1; i < r->vec_len; i++) mult[i] = 0; + assert(rank); while ((norm_str = pp2_charset_token_next(r->prt))) { - int res = word_entry_match(r->entries, norm_str); + int local_mult = 0; + int res = word_entry_match(r->entries, norm_str, rank, &local_mult); if (res) { assert(res < r->vec_len); - mult[res] += multiplier; + mult[res] += local_mult; } length++; } for (i = 1; i < r->vec_len; i++) { - if (length > 0) /* only add if non-empty */ + if (length > 0 && mult[i] > 0) /* only add if non-empty */ + { + wrbuf_printf(w, "%s: field=%s vecf[%d] += mult(%d) / length(%d);\n", + e->display_str, name, i, mult[i], length); cluster->term_frequency_vecf[i] += (double) mult[i] / length; + } cluster->term_frequency_vec[i] += mult[i]; + e = e->next; } cluster->term_frequency_vec[0] += length; @@ -107,7 +134,7 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) for (i = 0; i < numwords; i++) { const char *norm_str; - + ccl_field = nmem_strdup_null(res->nmem, n->u.t.qual); pp2_charset_token_first(res->prt, words[i], 0); @@ -120,6 +147,7 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) (*e)->norm_str = nmem_strdup(res->nmem, norm_str); (*e)->ccl_field = ccl_field; (*e)->termno = res->vec_len++; + (*e)->display_str = nmem_strdup(res->nmem, words[i]); (*e)->next = 0; } } @@ -130,7 +158,8 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) } struct relevance *relevance_create_ccl(pp2_charset_fact_t pft, - struct ccl_rpn_node *query) + struct ccl_rpn_node *query, + int rank_cluster) { NMEM nmem = nmem_create(); struct relevance *res = nmem_malloc(nmem, sizeof(*res)); @@ -139,13 +168,14 @@ struct relevance *relevance_create_ccl(pp2_charset_fact_t pft, res->nmem = nmem; res->entries = 0; res->vec_len = 1; + res->rank_cluster = rank_cluster; res->prt = pp2_charset_token_create(pft, "relevance"); - + pull_terms(res, query); res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int)); for (i = 0; i < res->vec_len; i++) - res->doc_frequency_vec[i] = 0; + res->doc_frequency_vec[i] = 0; return res; } @@ -171,14 +201,14 @@ void relevance_newrec(struct relevance *r, struct record_cluster *rec) r->vec_len * sizeof(*rec->term_frequency_vec)); for (i = 0; i < r->vec_len; i++) rec->term_frequency_vec[i] = 0; - + // term frequency divided by length of field [1,...] rec->term_frequency_vecf = nmem_malloc(r->nmem, r->vec_len * sizeof(*rec->term_frequency_vecf)); for (i = 0; i < r->vec_len; i++) rec->term_frequency_vecf[i] = 0.0; - + // for relevance_countwords (so we don't have to xmalloc/xfree) rec->term_frequency_vec_tmp = nmem_malloc(r->nmem, @@ -211,29 +241,51 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) idfvec[i] = 0; else { - // This conditional may be terribly wrong - // It was there to address the situation where vec[0] == vec[i] - // which leads to idfvec[i] == 0... not sure about this - // Traditional TF-IDF may assume that a word that occurs in every - // record is irrelevant, but this is actually something we will - // see a lot - if ((idfvec[i] = log((float) rel->doc_frequency_vec[0] / - rel->doc_frequency_vec[i])) < 0.0000001) - idfvec[i] = 1; + /* add one to nominator idf(t,D) to ensure a value > 0 */ + idfvec[i] = log((float) (1 + rel->doc_frequency_vec[0]) / + rel->doc_frequency_vec[i]); } } // Calculate relevance for each document while (1) { - int t; int relevance = 0; + WRBUF w; + struct word_entry *e = rel->entries; struct record_cluster *rec = reclist_read_record(reclist); if (!rec) break; - for (t = 1; t < rel->vec_len; t++) + w = rec->relevance_explain2; + wrbuf_rewind(w); + for (i = 1; i < rel->vec_len; i++) + { + float termfreq = (float) rec->term_frequency_vecf[i]; + int add = 100000 * termfreq * idfvec[i]; + + wrbuf_printf(w, "idf[%d] = log(((1 + total(%d))/termoccur(%d));\n", + i, rel->doc_frequency_vec[0], + rel->doc_frequency_vec[i]); + wrbuf_printf(w, "%s: relevance += 100000 * vecf[%d](%f) * " + "idf[%d](%f) (%d);\n", + e->display_str, i, termfreq, i, idfvec[i], add); + relevance += add; + e = e->next; + } + if (!rel->rank_cluster) + { + struct record *record; + int cluster_size = 0; + + for (record = rec->records; record; record = record->next) + cluster_size++; + + wrbuf_printf(w, "score = relevance(%d)/cluster_size(%d);\n", + relevance, cluster_size); + relevance /= cluster_size; + } + else { - float termfreq = (float) rec->term_frequency_vecf[t]; - relevance += 100000 * (termfreq * idfvec[t] + 0.0000005); + wrbuf_printf(w, "score = relevance(%d);\n", relevance); } rec->relevance_score = relevance; }