X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Fsession.c;h=0006652387ba9cf2b9aa2cb45f1e6ed3c1ed9dcc;hb=7e047ebfe0f7f78205a30a4474c1c33594aea1b3;hp=697df8573f0440c43f856952d8cd54cd44eebee4;hpb=9e89d4e3c7bc9c9a9b0e7cedc54ca3f15b49de1f;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index 697df85..0006652 100644 --- a/src/session.c +++ b/src/session.c @@ -95,6 +95,12 @@ struct client_list { struct client_list *next; }; +struct session_sorted_results { + const char *field; + int increasing; + struct session_sorted_results *next; +}; + /* session counting (1) , disable client counting (0) */ static YAZ_MUTEX g_session_mutex = 0; static int no_sessions = 0; @@ -246,13 +252,13 @@ static xmlDoc *record_to_xml(struct session *se, if (!rdoc) { session_log(se, YLOG_FATAL, "Non-wellformed XML received from %s", - db->url); + db->id); return 0; } if (global_parameters.dump_records) { - session_log(se, YLOG_LOG, "Un-normalized record from %s", db->url); + session_log(se, YLOG_LOG, "Un-normalized record from %s", db->id); log_xml_doc(rdoc); } @@ -340,7 +346,7 @@ static xmlDoc *normalize_record(struct session *se, if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) { session_log(se, YLOG_WARN, "Normalize failed from %s", - sdb->database->url); + sdb->database->id); } else { @@ -349,7 +355,7 @@ static xmlDoc *normalize_record(struct session *se, if (global_parameters.dump_records) { session_log(se, YLOG_LOG, "Normalized record from %s", - sdb->database->url); + sdb->database->id); log_xml_doc(rdoc); } } @@ -399,12 +405,8 @@ static int prepare_map(struct session *se, struct session_database *sdb) { const char *s; - if (!sdb->settings) - { - session_log(se, YLOG_WARN, "No settings on %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_XSLT))) + if (sdb->settings && sdb->settings[PZ_XSLT] && !sdb->map && + (s = session_setting_oneval(sdb, PZ_XSLT))) { char auto_stylesheet[256]; @@ -432,32 +434,13 @@ static int prepare_map(struct session *se, struct session_database *sdb) } } sdb->map = normalize_cache_get(se->normalize_cache, - se->service->server->config, s); + se->service, s); if (!sdb->map) return -1; } return 0; } -// This analyzes settings and recomputes any supporting data structures -// if necessary. -static int prepare_session_database(struct session *se, - struct session_database *sdb) -{ - if (!sdb->settings) - { - session_log(se, YLOG_WARN, - "No settings associated with %s", sdb->database->url); - return -1; - } - if (sdb->settings[PZ_XSLT] && !sdb->map) - { - if (prepare_map(se, sdb) < 0) - return -1; - } - return 0; -} - // called if watch should be removed because http_channel is to be destroyed static void session_watch_cancel(void *data, struct http_channel *c, void *data2) @@ -521,14 +504,29 @@ void session_alert_watch(struct session *s, int what) } //callback for grep_databases -static void select_targets_callback(void *context, struct session_database *db) +static void select_targets_callback(struct session *se, + struct session_database *db) { - struct session *se = (struct session*) context; - struct client *cl = client_create(); + struct client *cl; struct client_list *l; - client_set_database(cl, db); - client_set_session(cl, se); + for (l = se->clients_cached; l; l = l->next) + if (client_get_database(l->client) == db) + break; + + if (l) + cl = l->client; + else + { + cl = client_create(db->database->id); + client_set_database(cl, db); + client_set_session(cl, se); + + l = xmalloc(sizeof(*l)); + l->client = cl; + l->next = se->clients_cached; + se->clients_cached = l; + } l = xmalloc(sizeof(*l)); l->client = cl; @@ -548,6 +546,25 @@ static void session_remove_clients(struct session *se) while (l) { struct client_list *l_next = l->next; + xfree(l); + l = l_next; + } +} + +static void session_remove_cached_clients(struct session *se) +{ + struct client_list *l; + + session_remove_clients(se); + + session_enter(se); + l = se->clients_cached; + se->clients_cached = 0; + session_leave(se); + + while (l) + { + struct client_list *l_next = l->next; client_lock(l->client); client_set_session(l->client, 0); client_set_database(l->client, 0); @@ -590,12 +607,84 @@ int session_is_preferred_clients_ready(struct session *s) return res == 0; } -enum pazpar2_error_code search(struct session *se, - const char *query, - const char *startrecs, const char *maxrecs, - const char *filter, - const char *limit, - const char **addinfo) +static const char *get_strategy_plus_sort(struct client *l, const char *field) +{ + struct session_database *sdb = client_get_database(l); + struct setting *s; + + const char *strategy_plus_sort = 0; + + for (s = sdb->settings[PZ_SORTMAP]; s; s = s->next) + { + char *p = strchr(s->name + 3, ':'); + if (!p) + { + yaz_log(YLOG_WARN, "Malformed sortmap name: %s", s->name); + continue; + } + p++; + if (!strcmp(p, field)) + { + strategy_plus_sort = s->value; + break; + } + } + return strategy_plus_sort; +} + +void session_sort(struct session *se, const char *field, int increasing) +{ + struct session_sorted_results *sr; + struct client_list *l; + + session_enter(se); + + /* see if we already have sorted for this critieria */ + for (sr = se->sorted_results; sr; sr = sr->next) + { + if (!strcmp(field, sr->field) && increasing == sr->increasing) + break; + } + if (sr) + { + yaz_log(YLOG_LOG, "search_sort: field=%s increasing=%d already fetched", + field, increasing); + session_leave(se); + return; + } + yaz_log(YLOG_LOG, "search_sort: field=%s increasing=%d must fetch", + field, increasing); + sr = nmem_malloc(se->nmem, sizeof(*sr)); + sr->field = nmem_strdup(se->nmem, field); + sr->increasing = increasing; + sr->next = se->sorted_results; + se->sorted_results = sr; + + for (l = se->clients; l; l = l->next) + { + struct client *cl = l->client; + const char *strategy_plus_sort = get_strategy_plus_sort(cl, field); + if (strategy_plus_sort) + { + struct timeval tval; + if (client_prep_connection(cl, se->service->z3950_operation_timeout, + se->service->z3950_session_timeout, + se->service->server->iochan_man, + &tval)) + client_start_search(cl, strategy_plus_sort, increasing); + } + } + session_leave(se); +} + +enum pazpar2_error_code session_search(struct session *se, + const char *query, + const char *startrecs, + const char *maxrecs, + const char *filter, + const char *limit, + const char **addinfo, + const char *sort_field, int increasing) { int live_channels = 0; int no_working = 0; @@ -608,15 +697,26 @@ enum pazpar2_error_code search(struct session *se, *addinfo = 0; - session_remove_clients(se); + if (se->settings_modified) + session_remove_cached_clients(se); + else + session_remove_clients(se); session_enter(se); reclist_destroy(se->reclist); se->reclist = 0; + se->settings_modified = 0; relevance_destroy(&se->relevance); nmem_reset(se->nmem); - se->total_records = se->total_hits = se->total_merged = 0; + se->total_records = se->total_merged = 0; se->num_termlists = 0; + + /* reset list of sorted results and clear to relevance search */ + se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results)); + se->sorted_results->field = nmem_strdup(se->nmem, sort_field); + se->sorted_results->increasing = increasing; + se->sorted_results->next = 0; + live_channels = select_targets(se, filter); if (!live_channels) { @@ -638,24 +738,40 @@ enum pazpar2_error_code search(struct session *se, } for (l = se->clients; l; l = l->next) { + int parse_ret; struct client *cl = l->client; + const char *strategy_plus_sort = get_strategy_plus_sort(cl, sort_field); - if (maxrecs) - client_set_maxrecs(cl, atoi(maxrecs)); - if (startrecs) - client_set_startrecs(cl, atoi(startrecs)); - if (prepare_session_database(se, client_get_database(cl)) < 0) - ; - else if (client_parse_query(cl, query, facet_limits) < 0) + if (prepare_map(se, client_get_database(cl)) < 0) + continue; + + parse_ret = client_parse_query(cl, query, facet_limits, startrecs, + maxrecs); + if (parse_ret < 0) no_failed++; + else if (parse_ret == 0) + { + yaz_log(YLOG_LOG, "client NEW %s", client_get_id(cl)); + no_working++; + if (client_prep_connection(cl, se->service->z3950_operation_timeout, + se->service->z3950_session_timeout, + se->service->server->iochan_man, + &tval)) + client_start_search(cl, strategy_plus_sort, increasing); + } else { + yaz_log(YLOG_LOG, "client REUSE %s", client_get_id(cl)); no_working++; if (client_prep_connection(cl, se->service->z3950_operation_timeout, se->service->z3950_session_timeout, se->service->server->iochan_man, &tval)) - client_start_search(cl); + { + session_leave(se); + client_reingest(cl); + session_enter(se); + } } } facet_limits_destroy(facet_limits); @@ -670,6 +786,7 @@ enum pazpar2_error_code search(struct session *se, else return PAZPAR2_NO_TARGETS; } + yaz_log(YLOG_LOG, "session_start_search done"); return PAZPAR2_NO_ERROR; } @@ -717,8 +834,6 @@ static struct session_database *load_session_database(struct session *se, { struct database *db = new_database(id, se->session_nmem); - resolve_database(se->service, db); - session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list @@ -732,7 +847,7 @@ static struct session_database *find_session_database(struct session *se, struct session_database *sdb; for (sdb = se->databases; sdb; sdb = sdb->next) - if (!strcmp(sdb->database->url, id)) + if (!strcmp(sdb->database->id, id)) return sdb; return load_session_database(se, id); } @@ -755,6 +870,8 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, new->next = sdb->settings[offset]; sdb->settings[offset] = new; + se->settings_modified = 1; + // Force later recompute of settings-driven data structures // (happens when a search starts and client connections are prepared) switch (offset) @@ -768,11 +885,12 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, } } -void session_destroy(struct session *se) { +void session_destroy(struct session *se) +{ struct session_database *sdb; session_log(se, YLOG_DEBUG, "Destroying"); session_use(-1); - session_remove_clients(se); + session_remove_cached_clients(se); for (sdb = se->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); @@ -815,13 +933,14 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session_log(session, YLOG_DEBUG, "New"); session->service = service; session->relevance = 0; - session->total_hits = 0; session->total_records = 0; session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; session->num_termlists = 0; session->reclist = 0; session->clients = 0; + session->clients_cached = 0; + session->settings_modified = 0; session->session_nmem = nmem; session->nmem = nmem_create(); session->databases = 0; @@ -837,6 +956,8 @@ struct session *new_session(NMEM nmem, struct conf_service *service, return session; } +const char * client_get_suggestions_xml(struct client *cl, WRBUF wrbuf); + static struct hitsbytarget *hitsbytarget_nb(struct session *se, int *count, NMEM nmem) { @@ -856,7 +977,7 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_database(cl)->database->url; + res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); res[*count].records = client_get_num_records(cl); @@ -865,6 +986,9 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w)); + wrbuf_rewind(w); + wrbuf_puts(w, ""); + res[*count].suggestions_xml = nmem_strdup(nmem, client_get_suggestions_xml(cl, w)); wrbuf_destroy(w); (*count)++; } @@ -961,6 +1085,11 @@ void perform_termlist(struct http_channel *c, struct session *se, for (j = 0; j < num_names; j++) { const char *tname; + + wrbuf_puts(c->wrbuf, "wrbuf, names[j]); + wrbuf_puts(c->wrbuf, "\">\n"); + for (i = 0; i < se->num_termlists; i++) { tname = se->termlists[i].name; @@ -972,9 +1101,6 @@ void perform_termlist(struct http_channel *c, struct session *se, if (p) { int i; - wrbuf_puts(c->wrbuf, "wrbuf, tname); - wrbuf_puts(c->wrbuf, "\">\n"); for (i = 0; i < len && i < num; i++) { // prevent sending empty term elements @@ -991,19 +1117,15 @@ void perform_termlist(struct http_channel *c, struct session *se, p[i]->frequency); wrbuf_puts(c->wrbuf, "\n"); } - wrbuf_puts(c->wrbuf, "\n"); } } } tname = "xtargets"; if (num_names > 0 && !strcmp(names[j], tname)) { - wrbuf_puts(c->wrbuf, "wrbuf, tname); - wrbuf_puts(c->wrbuf, "\">\n"); targets_termlist_nb(c->wrbuf, se, num, c->nmem); - wrbuf_puts(c->wrbuf, "\n"); } + wrbuf_puts(c->wrbuf, "\n"); } session_leave(se); nmem_destroy(nmem_tmp); @@ -1076,6 +1198,8 @@ struct record_cluster **show_range_start(struct session *se, } else { + struct client_list *l; + for (spp = sp; spp; spp = spp->next) if (spp->type == Metadata_sortkey_relevance) { @@ -1086,7 +1210,10 @@ struct record_cluster **show_range_start(struct session *se, reclist_enter(se->reclist); *total = reclist_get_num_records(se->reclist); - *sumhits = se->total_hits; + + *sumhits = 0; + for (l = se->clients; l; l = l->next) + *sumhits += client_get_hits(l->client); for (i = 0; i < start; i++) if (!reclist_read_record(se->reclist)) @@ -1129,11 +1256,13 @@ void statistics(struct session *se, struct statistics *stat) int count = 0; memset(stat, 0, sizeof(*stat)); + stat->num_hits = 0; for (l = se->clients; l; l = l->next) { struct client *cl = l->client; if (!client_get_connection(cl)) stat->num_no_connection++; + stat->num_hits += client_get_hits(cl); switch (client_get_state(cl)) { case Client_Connecting: stat->num_connecting++; break; @@ -1145,7 +1274,6 @@ void statistics(struct session *se, struct statistics *stat) } count++; } - stat->num_hits = se->total_hits; stat->num_records = se->total_records; stat->num_clients = count; @@ -1305,8 +1433,13 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, /* generate unique key if none is not generated already or is empty */ if (wrbuf_len(norm_wr) == 0) { - wrbuf_printf(norm_wr, "%s-%d", - client_get_database(cl)->database->url, record_no); + wrbuf_printf(norm_wr, "position: %s-%d", + client_get_id(cl), record_no); + } + else + { + const char *lead = "content: "; + wrbuf_insert(norm_wr, 0, lead, strlen(lead)); } if (wrbuf_len(norm_wr) > 0) mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr)); @@ -1410,7 +1543,7 @@ int ingest_record(struct client *cl, const char *rec, if (!check_record_filter(root, sdb)) { session_log(se, YLOG_LOG, "Filtered out record no %d from %s", - record_no, sdb->database->url); + record_no, sdb->database->id); xmlFreeDoc(xdoc); return -2; } @@ -1424,7 +1557,7 @@ int ingest_record(struct client *cl, const char *rec, } session_enter(se); if (client_get_session(cl) == se) - ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); + ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); session_leave(se); xmlFreeDoc(xdoc); @@ -1440,43 +1573,18 @@ static int ingest_to_cluster(struct client *cl, xmlNode *n; xmlChar *type = 0; xmlChar *value = 0; - struct session_database *sdb = client_get_database(cl); struct session *se = client_get_session(cl); struct conf_service *service = se->service; + int term_factor = 1; + struct record_cluster *cluster; + struct session_database *sdb = client_get_database(cl); struct record *record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); - struct record_cluster *cluster = reclist_insert(se->reclist, - service, - record, - mergekey_norm, - &se->total_merged); - - const char *use_term_factor_str = session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); - int use_term_factor = 0; - int term_factor = 1; - if (use_term_factor_str && use_term_factor_str[0] != 0) - use_term_factor = atoi(use_term_factor_str); - if (use_term_factor) { - int maxrecs = client_get_maxrecs(cl); - int hits = (int) client_get_hits(cl); - term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); - assert(term_factor >= 1); - yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); - } - if (!cluster) - return -1; - if (global_parameters.dump_records) - session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, - sdb->database->url, record_no); - relevance_newrec(se->relevance, cluster); - - // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { - pp2_charset_token_t prt; if (type) xmlFree(type); if (value) @@ -1488,11 +1596,9 @@ static int ingest_to_cluster(struct client *cl, if (!strcmp((const char *) n->name, "metadata")) { struct conf_metadata *ser_md = 0; - struct conf_sortkey *ser_sk = 0; struct record_metadata **wheretoput = 0; struct record_metadata *rec_md = 0; int md_field_id = -1; - int sk_field_id = -1; type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); @@ -1512,13 +1618,8 @@ static int ingest_to_cluster(struct client *cl, se->number_of_warnings_unknown_metadata++; continue; } - + ser_md = &service->metadata[md_field_id]; - - if (ser_md->sortkey_offset >= 0){ - sk_field_id = ser_md->sortkey_offset; - ser_sk = &service->sortkeys[sk_field_id]; - } // non-merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, @@ -1533,10 +1634,80 @@ static int ingest_to_cluster(struct client *cl, while (*wheretoput) wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; + } + } + + cluster = reclist_insert(se->reclist, service, record, + mergekey_norm, &se->total_merged); + if (!cluster) + return -1; + + { + const char *use_term_factor_str = + session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); + if (use_term_factor_str && use_term_factor_str[0] == '1') + { + int maxrecs = client_get_maxrecs(cl); + int hits = (int) client_get_hits(cl); + term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); + assert(term_factor >= 1); + yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + } + } + + if (global_parameters.dump_records) + session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, + sdb->database->id, record_no); + + + relevance_newrec(se->relevance, cluster); + + // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) + { + pp2_charset_token_t prt; + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 1); + + if (!type || !value || !*value) + continue; + + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) + continue; + + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0) + { + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } // merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, ser_md->type, 0); + if (!rec_md) + continue; + wheretoput = &cluster->metadata[md_field_id]; // and polulate with data: