X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Fsession.c;h=fc021a87510c046ca378cc7968077213adce0324;hb=4d9f530309235e8c78d50f82196f135c3127c064;hp=bcd73535a2bf5c08c64ab0f36a246d7c8db8a9d0;hpb=0e0ddf5be474a96ae8838fe9845858b7ca96b9d8;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index bcd7353..fc021a8 100644 --- a/src/session.c +++ b/src/session.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2010 Index Data + Copyright (C) 2006-2011 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include +#include #include #include @@ -79,11 +80,14 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #define MAX_CHUNK 15 +#define MAX(a,b) ((a)>(b)?(a):(b)) + // Note: Some things in this structure will eventually move to configuration struct parameters global_parameters = { 0, // dump_records - 0 // debug_mode + 0, // debug_mode + 0, // predictable sessions }; struct client_list { @@ -91,6 +95,41 @@ struct client_list { struct client_list *next; }; +/* session counting (1) , disable client counting (0) */ +static YAZ_MUTEX g_session_mutex = 0; +static int no_sessions = 0; +static int no_session_total = 0; + +static int session_use(int delta) +{ + int sessions; + if (!g_session_mutex) + yaz_mutex_create(&g_session_mutex); + yaz_mutex_enter(g_session_mutex); + no_sessions += delta; + if (delta > 0) + no_session_total += delta; + sessions = no_sessions; + yaz_mutex_leave(g_session_mutex); + yaz_log(YLOG_DEBUG, "%s sesions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions); + return sessions; +} + +int sessions_count(void) { + return session_use(0); +} + +int session_count_total(void) { + int total = 0; + if (!g_session_mutex) + return 0; + yaz_mutex_enter(g_session_mutex); + total = no_session_total; + yaz_mutex_leave(g_session_mutex); + return total; +} + + static void log_xml_doc(xmlDoc *doc) { FILE *lf = yaz_log_file(); @@ -148,30 +187,82 @@ void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) void add_facet(struct session *s, const char *type, const char *value, int count) { + struct conf_service *service = s->service; + pp2_charset_token_t prt; + const char *facet_component; + WRBUF facet_wrbuf = wrbuf_alloc(); + WRBUF display_wrbuf = wrbuf_alloc(); int i; + const char *icu_chain_id = 0; - if (!*value) + for (i = 0; i < service->num_metadata; i++) + if (!strcmp((service->metadata + i)->name, type)) + icu_chain_id = (service->metadata + i)->facetrule; + yaz_log(YLOG_LOG, "icu_chain id=%s", icu_chain_id ? icu_chain_id : "null"); + + if (!icu_chain_id) + icu_chain_id = "facet"; + prt = pp2_charset_token_create(service->charsets, icu_chain_id); + if (!prt) + { + yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'", + icu_chain_id, type); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); return; - for (i = 0; i < s->num_termlists; i++) - if (!strcmp(s->termlists[i].name, type)) - break; - if (i == s->num_termlists) + } + pp2_charset_token_first(prt, value, 0); + while ((facet_component = pp2_charset_token_next(prt))) { - if (i == SESSION_MAX_TERMLISTS) + const char *display_component; + if (*facet_component) { - session_log(s, YLOG_FATAL, "Too many termlists"); - return; + if (wrbuf_len(facet_wrbuf)) + wrbuf_puts(facet_wrbuf, " "); + wrbuf_puts(facet_wrbuf, facet_component); + } + display_component = pp2_get_display(prt); + if (display_component) + { + if (wrbuf_len(display_wrbuf)) + wrbuf_puts(display_wrbuf, " "); + wrbuf_puts(display_wrbuf, display_component); } - - s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist - = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); - s->num_termlists = i + 1; } - session_log(s, YLOG_DEBUG, "Session: facets for %s: %s (%d)", - type, value, count); - - termlist_insert(s->termlists[i].termlist, value, count); + pp2_charset_token_destroy(prt); + + yaz_log(YLOG_LOG, "facet norm=%s", wrbuf_cstr(facet_wrbuf)); + yaz_log(YLOG_LOG, "facet display=%s", wrbuf_cstr(display_wrbuf)); + if (wrbuf_len(facet_wrbuf)) + { + int i; + for (i = 0; i < s->num_termlists; i++) + if (!strcmp(s->termlists[i].name, type)) + break; + if (i == s->num_termlists) + { + if (i == SESSION_MAX_TERMLISTS) + { + session_log(s, YLOG_FATAL, "Too many termlists"); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); + return; + } + + s->termlists[i].name = nmem_strdup(s->nmem, type); + s->termlists[i].termlist + = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); + s->num_termlists = i + 1; + } + +#if 0 + session_log(s, YLOG_DEBUG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); +#endif + termlist_insert(s->termlists[i].termlist, wrbuf_cstr(display_wrbuf), + wrbuf_cstr(facet_wrbuf), count); + } + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); } static xmlDoc *record_to_xml(struct session *se, @@ -427,7 +518,7 @@ int session_set_watch(struct session *s, int what, ret = 0; } session_leave(s); - return 0; + return ret; } void session_alert_watch(struct session *s, int what) @@ -452,7 +543,7 @@ void session_alert_watch(struct session *s, int what) session_leave(s); session_log(s, YLOG_DEBUG, - "session_alert_watch: %d calling function: %p", what, fun); + "Alert Watch: %d calling function: %p", what, fun); fun(data); } else @@ -468,6 +559,7 @@ static void select_targets_callback(void *context, struct session_database *db) client_set_database(cl, db); client_set_session(cl, se); + l = xmalloc(sizeof(*l)); l->client = cl; l->next = se->clients; @@ -524,16 +616,15 @@ int session_is_preferred_clients_ready(struct session *s) for (l = s->clients; l; l = l->next) if (client_is_active_preferred(l->client)) res++; - session_log(s, YLOG_DEBUG, "Session has %d active preferred clients.", res); + session_log(s, YLOG_DEBUG, "Has %d active preferred clients.", res); return res == 0; } - - enum pazpar2_error_code search(struct session *se, const char *query, const char *startrecs, const char *maxrecs, const char *filter, + const char *limit, const char **addinfo) { int live_channels = 0; @@ -541,6 +632,7 @@ enum pazpar2_error_code search(struct session *se, int no_failed = 0; struct client_list *l; struct timeval tval; + facet_limits_t facet_limits; session_log(se, YLOG_DEBUG, "Search"); @@ -567,6 +659,13 @@ enum pazpar2_error_code search(struct session *se, tval.tv_sec += 5; + facet_limits = facet_limits_create(limit); + if (!facet_limits) + { + *addinfo = "limit"; + session_leave(se); + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } for (l = se->clients; l; l = l->next) { struct client *cl = l->client; @@ -577,7 +676,7 @@ enum pazpar2_error_code search(struct session *se, client_set_startrecs(cl, atoi(startrecs)); if (prepare_session_database(se, client_get_database(cl)) < 0) ; - else if (client_parse_query(cl, query) < 0) + else if (client_parse_query(cl, query, facet_limits) < 0) no_failed++; else { @@ -589,6 +688,7 @@ enum pazpar2_error_code search(struct session *se, client_start_search(cl); } } + facet_limits_destroy(facet_limits); session_leave(se); if (no_working == 0) { @@ -698,11 +798,10 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, } } -void destroy_session(struct session *se) -{ +void session_destroy(struct session *se) { struct session_database *sdb; - - session_log(se, YLOG_DEBUG, "Pazpar2 session destroy"); + session_log(se, YLOG_DEBUG, "Destroying"); + session_use(-1); session_remove_clients(se); for (sdb = se->databases; sdb; sdb = sdb->next) @@ -713,9 +812,25 @@ void destroy_session(struct session *se) nmem_destroy(se->nmem); service_destroy(se->service); yaz_mutex_destroy(&se->session_mutex); - wrbuf_destroy(se->wrbuf); } +/* Depreciated: use session_destroy */ +void destroy_session(struct session *se) +{ + session_destroy(se); +} + +size_t session_get_memory_status(struct session *session) { + size_t session_nmem; + if (session == 0) + return 0; + session_enter(session); + session_nmem = nmem_total(session->nmem); + session_leave(session); + return session_nmem; +} + + struct session *new_session(NMEM nmem, struct conf_service *service, unsigned session_id) { @@ -727,7 +842,7 @@ struct session *new_session(NMEM nmem, struct conf_service *service, sprintf(tmp_str, "session#%u", session_id); session->session_id = session_id; - session_log(session, YLOG_DEBUG, "New session"); + session_log(session, YLOG_DEBUG, "New"); session->service = service; session->relevance = 0; session->total_hits = 0; @@ -739,7 +854,6 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->clients = 0; session->session_nmem = nmem; session->nmem = nmem_create(); - session->wrbuf = wrbuf_alloc(); session->databases = 0; for (i = 0; i <= SESSION_WATCH_MAX; i++) { @@ -749,7 +863,7 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->normalize_cache = normalize_cache_create(); session->session_mutex = 0; pazpar2_mutex_create(&session->session_mutex, tmp_str); - + session_use(1); return session; } @@ -780,14 +894,16 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); - res[*count].settings_xml = w; + res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w)); + wrbuf_destroy(w); (*count)++; } session_leave(se); return res; } -struct termlist_score **termlist(struct session *se, const char *name, int *num) +struct termlist_score **get_termlist_score(struct session *se, + const char *name, int *num) { int i; struct termlist_score **tl = 0; @@ -820,22 +936,25 @@ struct record_cluster *show_single_start(struct session *se, const char *id, struct record_cluster **prev_r, struct record_cluster **next_r) { - struct record_cluster *r; + struct record_cluster *r = 0; session_enter(se); - reclist_enter(se->reclist); *prev_r = 0; *next_r = 0; - while ((r = reclist_read_record(se->reclist))) + if (se->reclist) { - if (!strcmp(r->recid, id)) + reclist_enter(se->reclist); + while ((r = reclist_read_record(se->reclist))) { - *next_r = reclist_read_record(se->reclist); - break; + if (!strcmp(r->recid, id)) + { + *next_r = reclist_read_record(se->reclist); + break; + } + *prev_r = r; } - *prev_r = r; + reclist_leave(se->reclist); } - reclist_leave(se->reclist); if (!r) session_leave(se); return r; @@ -1016,15 +1135,15 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, if (value) { const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize(service->mergekey_pct); + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, "mergekey"); - pp2_relevance_first(prt, (const char *) value, 0); + pp2_charset_token_first(prt, (const char *) value, 0); if (wrbuf_len(norm_wr) > 0) wrbuf_puts(norm_wr, " "); wrbuf_puts(norm_wr, name); while ((norm_str = - pp2_relevance_token_next(prt))) + pp2_charset_token_next(prt))) { if (*norm_str) { @@ -1033,7 +1152,7 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, } } xmlFree(value); - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); no_found++; } } @@ -1055,11 +1174,11 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, if (mergekey) { const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize(service->mergekey_pct); + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, "mergekey"); - pp2_relevance_first(prt, (const char *) mergekey, 0); - while ((norm_str = pp2_relevance_token_next(prt))) + pp2_charset_token_first(prt, (const char *) mergekey, 0); + while ((norm_str = pp2_charset_token_next(prt))) { if (*norm_str) { @@ -1068,7 +1187,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, wrbuf_puts(norm_wr, norm_str); } } - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); xmlFree(mergekey); } else @@ -1180,6 +1299,7 @@ static int ingest_to_cluster(struct client *cl, \param nmem working NMEM \retval 0 OK \retval -1 failure + \retval -2 Filtered */ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem) @@ -1199,10 +1319,10 @@ int ingest_record(struct client *cl, const char *rec, if (!check_record_filter(root, sdb)) { - session_log(se, YLOG_WARN, "Filtered out record no %d from %s", + session_log(se, YLOG_LOG, "Filtered out record no %d from %s", record_no, sdb->database->url); xmlFreeDoc(xdoc); - return -1; + return -2; } mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem); @@ -1242,6 +1362,20 @@ static int ingest_to_cluster(struct client *cl, record, mergekey_norm, &se->total_merged); + + const char *use_term_factor_str = session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); + int use_term_factor = 0; + int term_factor = 1; + if (use_term_factor_str && use_term_factor_str[0] != 0) + use_term_factor = atoi(use_term_factor_str); + if (use_term_factor) { + int maxrecs = client_get_maxrecs(cl); + int hits = (int) client_get_hits(cl); + term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); + assert(term_factor >= 1); + yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + } + if (!cluster) return -1; if (global_parameters.dump_records) @@ -1252,7 +1386,7 @@ static int ingest_to_cluster(struct client *cl, // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { - pp2_relevance_token_t prt; + pp2_charset_token_t prt; if (type) xmlFree(type); if (value) @@ -1319,16 +1453,15 @@ static int ingest_to_cluster(struct client *cl, // assign cluster or record based on merge action if (ser_md->merge == Metadata_merge_unique) { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text.disp, + while (*wheretoput) + { + if (!strcmp((const char *) (*wheretoput)->data.text.disp, rec_md->data.text.disp)) break; - if (!mnode) - { - rec_md->next = *wheretoput; - *wheretoput = rec_md; + wheretoput = &(*wheretoput)->next; } + if (!*wheretoput) + *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_longest) { @@ -1348,12 +1481,13 @@ static int ingest_to_cluster(struct client *cl, nmem_malloc(se->nmem, sizeof(union data_types)); - prt = pp2_relevance_tokenize(service->sort_pct); + prt = + pp2_charset_token_create(service->charsets, "sort"); - pp2_relevance_first(prt, rec_md->data.text.disp, - skip_article); + pp2_charset_token_first(prt, rec_md->data.text.disp, + skip_article); - pp2_relevance_token_next(prt); + pp2_charset_token_next(prt); sort_str = pp2_get_sort(prt); @@ -1367,13 +1501,14 @@ static int ingest_to_cluster(struct client *cl, } cluster->sortkeys[sk_field_id]->text.sort = nmem_strdup(se->nmem, sort_str); - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); } } } else if (ser_md->merge == Metadata_merge_all) { - rec_md->next = *wheretoput; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_range) @@ -1406,20 +1541,20 @@ static int ingest_to_cluster(struct client *cl, // construct facets ... unless the client already has reported them if (ser_md->termlist && !client_has_facet(cl, (char *) type)) { - if (ser_md->type == Metadata_type_year) { char year[64]; sprintf(year, "%d", rec_md->data.number.max); - add_facet(se, (char *) type, year, 1); + + add_facet(se, (char *) type, year, term_factor); if (rec_md->data.number.max != rec_md->data.number.min) { sprintf(year, "%d", rec_md->data.number.min); - add_facet(se, (char *) type, year, 1); + add_facet(se, (char *) type, year, term_factor); } } else - add_facet(se, (char *) type, (char *) value, 1); + add_facet(se, (char *) type, (char *) value, term_factor); } // cleaning up @@ -1453,7 +1588,7 @@ void session_log(struct session *s, int level, const char *fmt, ...) va_start(ap, fmt); yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap); - yaz_log(level, "%u %s", s->session_id, buf); + yaz_log(level, "Session (%u): %s", s->session_id, buf); va_end(ap); }