From 7e047ebfe0f7f78205a30a4474c1c33594aea1b3 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Wed, 2 Nov 2011 15:24:39 +0100 Subject: [PATCH] Clients cached within session Clients are reused if query and other parameters are same for command=search. For example, if a subset of databases are used and query is the same, the result set from previous search is reused. --- src/client.c | 140 ++++++++++++++++++++++++++++++++++----------------------- src/client.h | 7 +-- src/session.c | 113 ++++++++++++++++++++++++++++++---------------- src/session.h | 2 + 4 files changed, 163 insertions(+), 99 deletions(-) diff --git a/src/client.c b/src/client.c index e17e604..ea171b9 100644 --- a/src/client.c +++ b/src/client.c @@ -554,6 +554,52 @@ void client_got_records(struct client *cl) } } +static void client_record_ingest(struct client *cl) +{ + const char *msg, *addinfo; + ZOOM_record rec = 0; + ZOOM_resultset resultset = cl->resultset; + int offset = cl->record_offset; + if ((rec = ZOOM_resultset_record(resultset, offset))) + { + cl->record_offset++; + if (cl->session == 0) + ; + else if (ZOOM_record_error(rec, &msg, &addinfo, 0)) + { + yaz_log(YLOG_WARN, "Record error %s (%s): %s (rec #%d)", + msg, addinfo, client_get_id(cl), + cl->record_offset); + } + else + { + struct session_database *sdb = client_get_database(cl); + NMEM nmem = nmem_create(); + const char *xmlrec; + char type[80]; + + if (nativesyntax_to_type(sdb, type, rec)) + yaz_log(YLOG_WARN, "Failed to determine record type"); + xmlrec = ZOOM_record_get(rec, type, NULL); + if (!xmlrec) + yaz_log(YLOG_WARN, "ZOOM_record_get failed from %s", + client_get_id(cl)); + else + { + /* OK = 0, -1 = failure, -2 = Filtered */ + if (ingest_record(cl, xmlrec, cl->record_offset, nmem) == -1) + yaz_log(YLOG_WARN, "Failed to ingest from %s", client_get_id(cl)); + } + nmem_destroy(nmem); + } + } + else + { + yaz_log(YLOG_WARN, "Expected record, but got NULL, offset=%d", + offset); + } +} + void client_record_response(struct client *cl) { struct connection *co = cl->connection; @@ -569,11 +615,9 @@ void client_record_response(struct client *cl) } else { - ZOOM_record rec = 0; - const char *msg, *addinfo; - if (cl->show_raw && cl->show_raw->active) { + ZOOM_record rec = 0; if ((rec = ZOOM_resultset_record(resultset, cl->show_raw->position-1))) { @@ -588,49 +632,21 @@ void client_record_response(struct client *cl) } else { - int offset = cl->record_offset; - if ((rec = ZOOM_resultset_record(resultset, offset))) - { - cl->record_offset++; - if (cl->session == 0) - ; - else if (ZOOM_record_error(rec, &msg, &addinfo, 0)) - { - yaz_log(YLOG_WARN, "Record error %s (%s): %s (rec #%d)", - msg, addinfo, client_get_id(cl), - cl->record_offset); - } - else - { - struct session_database *sdb = client_get_database(cl); - NMEM nmem = nmem_create(); - const char *xmlrec; - char type[80]; - - if (nativesyntax_to_type(sdb, type, rec)) - yaz_log(YLOG_WARN, "Failed to determine record type"); - xmlrec = ZOOM_record_get(rec, type, NULL); - if (!xmlrec) - yaz_log(YLOG_WARN, "ZOOM_record_get failed from %s", - client_get_id(cl)); - else - { - /* OK = 0, -1 = failure, -2 = Filtered */ - if (ingest_record(cl, xmlrec, cl->record_offset, nmem) == -1) - yaz_log(YLOG_WARN, "Failed to ingest from %s", client_get_id(cl)); - } - nmem_destroy(nmem); - } - } - else - { - yaz_log(YLOG_WARN, "Expected record, but got NULL, offset=%d", - offset); - } + client_record_ingest(cl); } } } +void client_reingest(struct client *cl) +{ + int i = cl->startrecs; + int to = cl->record_offset; + + cl->record_offset = i; + for (; i < to; i++) + client_record_ingest(cl); +} + static void client_set_facets_request(struct client *cl, ZOOM_connection link) { struct session_database *sdb = client_get_database(cl); @@ -695,6 +711,7 @@ void client_start_search(struct client *cl, const char *sort_strategy_and_spec, assert(link); + cl->hits = 0; cl->record_offset = 0; cl->diagnostic = 0; @@ -1036,7 +1053,8 @@ static void apply_limit(struct session_database *sdb, // Parse the query given the settings specific to this client int client_parse_query(struct client *cl, const char *query, - facet_limits_t facet_limits) + facet_limits_t facet_limits, + const char *startrecs, const char *maxrecs) { struct session *se = client_get_session(cl); struct session_database *sdb = client_get_database(cl); @@ -1048,10 +1066,24 @@ int client_parse_query(struct client *cl, const char *query, const char *pqf_strftime = session_setting_oneval(sdb, PZ_PQF_STRFTIME); const char *query_syntax = session_setting_oneval(sdb, PZ_QUERY_SYNTAX); WRBUF w_ccl, w_pqf; + int ret_value = 1; + if (!ccl_map) return -1; - cl->hits = -1; + + if (maxrecs && atoi(maxrecs) != cl->maxrecs) + { + ret_value = 0; + cl->maxrecs = atoi(maxrecs); + } + + if (startrecs && atoi(startrecs) != cl->startrecs) + { + ret_value = 0; + cl->startrecs = atoi(startrecs); + } + w_ccl = wrbuf_alloc(); wrbuf_puts(w_ccl, query); @@ -1099,8 +1131,12 @@ int client_parse_query(struct client *cl, const char *query, wrbuf_putc(w_pqf, cp[0]); } } - xfree(cl->pquery); - cl->pquery = xstrdup(wrbuf_cstr(w_pqf)); + if (!cl->pquery || strcmp(cl->pquery, wrbuf_cstr(w_pqf))) + { + xfree(cl->pquery); + cl->pquery = xstrdup(wrbuf_cstr(w_pqf)); + ret_value = 0; + } wrbuf_destroy(w_pqf); yaz_log(YLOG_LOG, "PQF query: %s", cl->pquery); @@ -1133,7 +1169,7 @@ int client_parse_query(struct client *cl, const char *query, } ccl_rpn_delete(cn); - return 0; + return ret_value; } void client_set_session(struct client *cl, struct session *se) @@ -1222,21 +1258,11 @@ const char *client_get_id(struct client *cl) return cl->id; } -void client_set_maxrecs(struct client *cl, int v) -{ - cl->maxrecs = v; -} - int client_get_maxrecs(struct client *cl) { return cl->maxrecs; } -void client_set_startrecs(struct client *cl, int v) -{ - cl->startrecs = v; -} - void client_set_preferred(struct client *cl, int v) { cl->preferred = v; diff --git a/src/client.h b/src/client.h index 2fb8ada..2086871 100644 --- a/src/client.h +++ b/src/client.h @@ -84,16 +84,15 @@ int client_is_active_preferred(struct client *cl); struct client *client_next_in_session(struct client *cl); int client_parse_query(struct client *cl, const char *query, - facet_limits_t facet_limits); + facet_limits_t facet_limits, const char *startrecs, + const char *maxrecs); Odr_int client_get_hits(struct client *cl); int client_get_num_records(struct client *cl); int client_get_diagnostic(struct client *cl); void client_set_diagnostic(struct client *cl, int diagnostic); void client_set_database(struct client *cl, struct session_database *db); const char *client_get_id(struct client *cl); -void client_set_maxrecs(struct client *cl, int v); int client_get_maxrecs(struct client *cl); -void client_set_startrecs(struct client *cl, int v); void client_remove_from_session(struct client *c); void client_incref(struct client *c); void client_got_records(struct client *c); @@ -102,6 +101,8 @@ void client_unlock(struct client *c); int client_has_facet(struct client *cl, const char *name); void client_check_preferred_watch(struct client *cl); +void client_reingest(struct client *cl); + #endif diff --git a/src/session.c b/src/session.c index 439ad7b..0006652 100644 --- a/src/session.c +++ b/src/session.c @@ -405,12 +405,8 @@ static int prepare_map(struct session *se, struct session_database *sdb) { const char *s; - if (!sdb->settings) - { - session_log(se, YLOG_WARN, "No settings on %s", sdb->database->id); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_XSLT))) + if (sdb->settings && sdb->settings[PZ_XSLT] && !sdb->map && + (s = session_setting_oneval(sdb, PZ_XSLT))) { char auto_stylesheet[256]; @@ -445,25 +441,6 @@ static int prepare_map(struct session *se, struct session_database *sdb) return 0; } -// This analyzes settings and recomputes any supporting data structures -// if necessary. -static int prepare_session_database(struct session *se, - struct session_database *sdb) -{ - if (!sdb->settings) - { - session_log(se, YLOG_WARN, - "No settings associated with %s", sdb->database->id); - return -1; - } - if (sdb->settings[PZ_XSLT] && !sdb->map) - { - if (prepare_map(se, sdb) < 0) - return -1; - } - return 0; -} - // called if watch should be removed because http_channel is to be destroyed static void session_watch_cancel(void *data, struct http_channel *c, void *data2) @@ -530,12 +507,26 @@ void session_alert_watch(struct session *s, int what) static void select_targets_callback(struct session *se, struct session_database *db) { - struct client *cl = client_create(db->database->id); + struct client *cl; struct client_list *l; - client_set_database(cl, db); + for (l = se->clients_cached; l; l = l->next) + if (client_get_database(l->client) == db) + break; - client_set_session(cl, se); + if (l) + cl = l->client; + else + { + cl = client_create(db->database->id); + client_set_database(cl, db); + client_set_session(cl, se); + + l = xmalloc(sizeof(*l)); + l->client = cl; + l->next = se->clients_cached; + se->clients_cached = l; + } l = xmalloc(sizeof(*l)); l->client = cl; @@ -555,6 +546,25 @@ static void session_remove_clients(struct session *se) while (l) { struct client_list *l_next = l->next; + xfree(l); + l = l_next; + } +} + +static void session_remove_cached_clients(struct session *se) +{ + struct client_list *l; + + session_remove_clients(se); + + session_enter(se); + l = se->clients_cached; + se->clients_cached = 0; + session_leave(se); + + while (l) + { + struct client_list *l_next = l->next; client_lock(l->client); client_set_session(l->client, 0); client_set_database(l->client, 0); @@ -687,11 +697,15 @@ enum pazpar2_error_code session_search(struct session *se, *addinfo = 0; - session_remove_clients(se); + if (se->settings_modified) + session_remove_cached_clients(se); + else + session_remove_clients(se); session_enter(se); reclist_destroy(se->reclist); se->reclist = 0; + se->settings_modified = 0; relevance_destroy(&se->relevance); nmem_reset(se->nmem); se->total_records = se->total_merged = 0; @@ -724,19 +738,20 @@ enum pazpar2_error_code session_search(struct session *se, } for (l = se->clients; l; l = l->next) { + int parse_ret; struct client *cl = l->client; const char *strategy_plus_sort = get_strategy_plus_sort(cl, sort_field); - if (maxrecs) - client_set_maxrecs(cl, atoi(maxrecs)); - if (startrecs) - client_set_startrecs(cl, atoi(startrecs)); - if (prepare_session_database(se, client_get_database(cl)) < 0) - ; - else if (client_parse_query(cl, query, facet_limits) < 0) + if (prepare_map(se, client_get_database(cl)) < 0) + continue; + + parse_ret = client_parse_query(cl, query, facet_limits, startrecs, + maxrecs); + if (parse_ret < 0) no_failed++; - else + else if (parse_ret == 0) { + yaz_log(YLOG_LOG, "client NEW %s", client_get_id(cl)); no_working++; if (client_prep_connection(cl, se->service->z3950_operation_timeout, se->service->z3950_session_timeout, @@ -744,6 +759,20 @@ enum pazpar2_error_code session_search(struct session *se, &tval)) client_start_search(cl, strategy_plus_sort, increasing); } + else + { + yaz_log(YLOG_LOG, "client REUSE %s", client_get_id(cl)); + no_working++; + if (client_prep_connection(cl, se->service->z3950_operation_timeout, + se->service->z3950_session_timeout, + se->service->server->iochan_man, + &tval)) + { + session_leave(se); + client_reingest(cl); + session_enter(se); + } + } } facet_limits_destroy(facet_limits); session_leave(se); @@ -757,6 +786,7 @@ enum pazpar2_error_code session_search(struct session *se, else return PAZPAR2_NO_TARGETS; } + yaz_log(YLOG_LOG, "session_start_search done"); return PAZPAR2_NO_ERROR; } @@ -840,6 +870,8 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, new->next = sdb->settings[offset]; sdb->settings[offset] = new; + se->settings_modified = 1; + // Force later recompute of settings-driven data structures // (happens when a search starts and client connections are prepared) switch (offset) @@ -853,11 +885,12 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, } } -void session_destroy(struct session *se) { +void session_destroy(struct session *se) +{ struct session_database *sdb; session_log(se, YLOG_DEBUG, "Destroying"); session_use(-1); - session_remove_clients(se); + session_remove_cached_clients(se); for (sdb = se->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); @@ -906,6 +939,8 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->num_termlists = 0; session->reclist = 0; session->clients = 0; + session->clients_cached = 0; + session->settings_modified = 0; session->session_nmem = nmem; session->nmem = nmem_create(); session->databases = 0; diff --git a/src/session.h b/src/session.h index 13bc314..e58302b 100644 --- a/src/session.h +++ b/src/session.h @@ -100,6 +100,7 @@ struct session { struct conf_service *service; /* service in use for this session */ struct session_database *databases; // All databases, settings overriden struct client_list *clients; // Clients connected for current search + struct client_list *clients_cached; // Clients in cache NMEM session_nmem; // Nmem for session-permanent storage NMEM nmem; // Nmem for each operation (i.e. search, result set, etc) int num_termlists; @@ -114,6 +115,7 @@ struct session { normalize_cache_t normalize_cache; YAZ_MUTEX session_mutex; unsigned session_id; + int settings_modified; struct session_sorted_results *sorted_results; }; -- 1.7.10.4