X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Flogic.c;h=09fb3fd96505eb7a2ce82dc6543c8bd2f1d9c5a2;hb=b2807317725db68d786503711be67ecf163115b7;hp=ee30f508e2e2a4c5ed2839d33167c503d9c171a3;hpb=7d0d04af0ae9a019adf0ee16248b5e9c8ea21922;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index ee30f50..09fb3fd 100644 --- a/src/logic.c +++ b/src/logic.c @@ -57,7 +57,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #endif - +#include "parameters.h" #include "pazpar2.h" #include "eventl.h" #include "http.h" @@ -68,6 +68,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "client.h" #include "settings.h" #include "normalize7bit.h" +#include "marcmap.h" #define TERMLIST_HIGH_SCORE 25 @@ -78,11 +79,7 @@ struct parameters global_parameters = { 0, // dump_records 0, // debug_mode - 30, // operations timeout - 60, // session timeout 100, - 180, // Z39.50 session timeout - 15 // Connect timeout }; static void log_xml_doc(xmlDoc *doc) @@ -261,8 +258,17 @@ xmlDoc *normalize_record(struct session_database *sdb, struct session *se, insert_settings_parameters(sdb, se, parms); - new = xsltApplyStylesheet(m->stylesheet, rdoc, (const char **) parms); - root= xmlDocGetRootElement(new); + if (m->stylesheet) + { + new = xsltApplyStylesheet(m->stylesheet, rdoc, (const char **) parms); + } + else if (m->marcmap) + { + new = marcmap_apply(m->marcmap, rdoc); + } + + root = xmlDocGetRootElement(new); + if (!new || !root || !(root->children)) { yaz_log(YLOG_WARN, "XSLT transformation failed from %s", @@ -344,15 +350,37 @@ static int prepare_map(struct session *se, struct session_database *sdb) nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num); for (i = 0; i < num; i++) { + WRBUF fname = conf_get_fname(se->service, stylesheets[i]); + (*m) = nmem_malloc(se->session_nmem, sizeof(**m)); (*m)->next = 0; - if (!((*m)->stylesheet = conf_load_stylesheet(se->service->config, - stylesheets[i]))) + + // XSLT + if (!strcmp(&stylesheets[i][strlen(stylesheets[i])-4], ".xsl")) + { + (*m)->marcmap = NULL; + if (!((*m)->stylesheet = + xsltParseStylesheetFile((xmlChar *) wrbuf_cstr(fname)))) + { + yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load stylesheet: %s", + stylesheets[i]); + wrbuf_destroy(fname); + return -1; + } + } + // marcmap + else if (!strcmp(&stylesheets[i][strlen(stylesheets[i])-5], ".mmap")) { - yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load stylesheet: %s", - stylesheets[i]); - return -1; + (*m)->stylesheet = NULL; + if (!((*m)->marcmap = marcmap_load(wrbuf_cstr(fname), se->session_nmem))) + { + yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load marcmap: %s", + stylesheets[i]); + wrbuf_destroy(fname); + return -1; + } } + wrbuf_destroy(fname); m = &(*m)->next; } } @@ -477,10 +505,14 @@ static struct database_criterion *parse_filter(NMEM m, const char *buf) int subnum; int subi; struct database_criterion *new = nmem_malloc(m, sizeof(*new)); - char *eq = strchr(values[i], '='); + char *eq; + if ((eq = strchr(values[i], '='))) + new->type = PAZPAR2_STRING_MATCH; + if ((eq = strchr(values[i], '~'))) + new->type = PAZPAR2_SUBSTRING_MATCH; if (!eq) { - yaz_log(YLOG_WARN, "Missing equal-sign in filter"); + yaz_log(YLOG_WARN, "Missing equal-signi/tilde in filter"); return 0; } *(eq++) = '\0'; @@ -502,7 +534,7 @@ static struct database_criterion *parse_filter(NMEM m, const char *buf) } enum pazpar2_error_code search(struct session *se, - char *query, char *filter, + const char *query, const char *filter, const char **addinfo) { int live_channels = 0; @@ -543,7 +575,8 @@ enum pazpar2_error_code search(struct session *se, else { no_working++; - if (client_prep_connection(cl)) + if (client_prep_connection(cl, se->service->z3950_operation_timeout, + se->service->z3950_session_timeout)) client_start_search(cl); } } @@ -606,6 +639,8 @@ static struct session_database *load_session_database(struct session *se, { struct database *db = find_database(id, 0, se->service); + resolve_database(db); + session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list return se->databases; @@ -676,6 +711,7 @@ void destroy_session(struct session *s) for (sdb = s->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); nmem_destroy(s->nmem); + service_destroy(s->service); wrbuf_destroy(s->wrbuf); } @@ -870,7 +906,8 @@ void statistics(struct session *se, struct statistics *stat) // Master list of connections we're handling events to -static IOCHAN channel_list = 0; +static IOCHAN channel_list = 0; /* thread pr */ + void pazpar2_add_channel(IOCHAN chan) { chan->next = channel_list; @@ -912,15 +949,60 @@ static struct record_metadata *record_metadata_init( return rec_md; } +static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, + struct conf_service *service, WRBUF norm_wr) +{ + xmlNode *n; + int no_found = 0; + for (n = root->children; n; n = n->next) + { + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + if (!strcmp(name, (const char *) type)) + { + xmlChar *value = xmlNodeListGetString(doc, n->children, 1); + if (value) + { + const char *norm_str; + pp2_relevance_token_t prt = + pp2_relevance_tokenize( + service->mergekey_pct, + (const char *) value); + + wrbuf_puts(norm_wr, name); + wrbuf_puts(norm_wr, "="); + while ((norm_str = + pp2_relevance_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + xmlFree(value); + pp2_relevance_token_destroy(prt); + no_found++; + } + } + xmlFree(type); + } + } + return no_found; +} + static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, struct conf_service *service, NMEM nmem) { char *mergekey_norm = 0; xmlNode *root = xmlDocGetRootElement(doc); WRBUF norm_wr = wrbuf_alloc(); - xmlNode *n; - /* create mergekey based on mergekey attribute from XSL (if any) */ + /* consider mergekey from XSL first */ xmlChar *mergekey = xmlGetProp(root, (xmlChar *) "mergekey"); if (mergekey) { @@ -942,53 +1024,25 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, pp2_relevance_token_destroy(prt); xmlFree(mergekey); } - /* append (if any) mergekey=yes metadata values */ - for (n = root->children; n; n = n->next) + else { - if (n->type != XML_ELEMENT_NODE) - continue; - if (!strcmp((const char *) n->name, "metadata")) + /* no mergekey defined in XSL. Look for mergekey metadata instead */ + int field_id; + for (field_id = 0; field_id < service->num_metadata; field_id++) { - struct conf_metadata *ser_md = 0; - int md_field_id = -1; - - xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); - - if (!type) - continue; - - md_field_id - = conf_service_metadata_field_id(service, - (const char *) type); - if (md_field_id >= 0) + struct conf_metadata *ser_md = &service->metadata[field_id]; + if (ser_md->mergekey != Metadata_mergekey_no) { - ser_md = &service->metadata[md_field_id]; - if (ser_md->mergekey == Metadata_mergekey_yes) + int r = get_mergekey_from_doc(doc, root, ser_md->name, + service, norm_wr); + if (r == 0 && ser_md->mergekey == Metadata_mergekey_required) { - xmlChar *value = xmlNodeListGetString(doc, n->children, 1); - if (value) - { - const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize( - service->mergekey_pct, - (const char *) value); - - while ((norm_str = pp2_relevance_token_next(prt))) - { - if (*norm_str) - { - if (wrbuf_len(norm_wr)) - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - xmlFree(value); - pp2_relevance_token_destroy(prt); - } + /* no mergekey on this one and it is required.. + Generate unique key instead */ + wrbuf_rewind(norm_wr); + break; } } - xmlFree(type); } } @@ -1004,6 +1058,58 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, return mergekey_norm; } +/** \brief see if metadata for pz:recordfilter exists + \param root xml root element of normalized record + \param sdb session database for client + \retval 0 if there is no metadata for pz:recordfilter + \retval 1 if there is metadata for pz:recordfilter + + If there is no pz:recordfilter defined, this function returns 1 + as well. +*/ + +static int check_record_filter(xmlNode *root, struct session_database *sdb) +{ + int match = 0; + xmlNode *n; + const char *s; + s = session_setting_oneval(sdb, PZ_RECORDFILTER); + + if (!s || !*s) + return 1; + + for (n = root->children; n; n = n->next) + { + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + if (type) + { + size_t len; + const char *eq = strchr(s, '='); + if (eq) + len = eq - s; + else + len = strlen(s); + if (len == strlen((const char *)type) && + !memcmp((const char *) type, s, len)) + { + xmlChar *value = xmlNodeGetContent(n); + if (value && *value) + { + if (!eq || strstr((const char *) value, eq+1)) + match = 1; + } + xmlFree(value); + } + xmlFree(type); + } + } + } + return match; +} /** \brief ingest XML record @@ -1015,12 +1121,12 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, struct record *ingest_record(struct client *cl, const char *rec, int record_no) { - xmlDoc *xdoc = normalize_record(client_get_database(cl), - client_get_session(cl), rec); + struct session_database *sdb = client_get_database(cl); + struct session *se = client_get_session(cl); + xmlDoc *xdoc = normalize_record(sdb, se, rec); xmlNode *root, *n; struct record *record; struct record_cluster *cluster; - struct session *se = client_get_session(cl); const char *mergekey_norm; xmlChar *type = 0; xmlChar *value = 0; @@ -1031,6 +1137,14 @@ struct record *ingest_record(struct client *cl, const char *rec, root = xmlDocGetRootElement(xdoc); + if (!check_record_filter(root, sdb)) + { + yaz_log(YLOG_WARN, "Filtered out record no %d from %s", record_no, + sdb->database->url); + xmlFreeDoc(xdoc); + return 0; + } + mergekey_norm = get_mergekey(xdoc, cl, record_no, service, se->nmem); if (!mergekey_norm) { @@ -1048,7 +1162,7 @@ struct record *ingest_record(struct client *cl, const char *rec, &se->total_merged); if (global_parameters.dump_records) yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, - client_get_database(cl)->database->url, record_no); + sdb->database->url, record_no); if (!cluster) { /* no room for record */