X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Flogic.c;h=23d60dc32ad7861f4fc57a0d285b66150ac6afd8;hb=a8f5308820cd689f5b92601038cb5d24118d5f2b;hp=b3d2a2bcf439383957912f7f0070ef8c9823cf83;hpb=dbcf2c3410ed74d919efdc62ffbd00e716284d8e;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index b3d2a2b..23d60dc 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,4 +1,4 @@ -/* $Id: logic.c,v 1.20 2007-04-23 21:05:23 adam Exp $ +/* $Id: logic.c,v 1.25 2007-04-26 21:33:32 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -70,6 +70,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "database.h" #include "client.h" #include "settings.h" +#include "normalize7bit.h" #define MAX_CHUNK 15 @@ -121,75 +122,6 @@ void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) } } -char *normalize_mergekey(char *buf, int skiparticle) -{ - char *p = buf, *pout = buf; - - if (skiparticle) - { - char firstword[64]; - char articles[] = "the den der die des an a "; // must end in space - - while (*p && !isalnum(*p)) - p++; - pout = firstword; - while (*p && *p != ' ' && pout - firstword < 62) - *(pout++) = tolower(*(p++)); - *(pout++) = ' '; - *(pout++) = '\0'; - if (!strstr(articles, firstword)) - p = buf; - pout = buf; - } - - while (*p) - { - while (*p && !isalnum(*p)) - p++; - while (isalnum(*p)) - *(pout++) = tolower(*(p++)); - if (*p) - *(pout++) = ' '; - while (*p && !isalnum(*p)) - p++; - } - if (buf != pout) - do { - *(pout--) = '\0'; - } - while (pout > buf && *pout == ' '); - - return buf; -} - -// Extract what appears to be years from buf, storing highest and -// lowest values. -static int extract_years(const char *buf, int *first, int *last) -{ - *first = -1; - *last = -1; - while (*buf) - { - const char *e; - int len; - - while (*buf && !isdigit(*buf)) - buf++; - len = 0; - for (e = buf; *e && isdigit(*e); e++) - len++; - if (len == 4) - { - int value = atoi(buf); - if (*first < 0 || value < *first) - *first = value; - if (*last < 0 || value > *last) - *last = value; - } - buf = e; - } - return *first; -} static void add_facet(struct session *s, const char *type, const char *value) @@ -210,7 +142,8 @@ static void add_facet(struct session *s, const char *type, const char *value) } s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist = termlist_create(s->nmem, s->expected_maxrecs, 15); + s->termlists[i].termlist + = termlist_create(s->nmem, s->expected_maxrecs, 15); s->num_termlists = i + 1; } termlist_insert(s->termlists[i].termlist, value); @@ -290,9 +223,7 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) return 0; } } -#endif - -#if 0 +#else // do it another way to detect transformation errors right now // but does not seem to work either! { @@ -347,7 +278,8 @@ static int prepare_yazmarc(struct session_database *sdb) yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); return -1; } - if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) && !strncmp(s, "iso2709", 7)) + if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) + && !strncmp(s, "iso2709", 7)) { char *encoding = "marc-8s", *e; yaz_iconv_t cm; @@ -414,11 +346,13 @@ static int prepare_map(struct session *se, struct session_database *sdb) // This analyzes settings and recomputes any supporting data structures // if necessary. -static int prepare_session_database(struct session *se, struct session_database *sdb) +static int prepare_session_database(struct session *se, + struct session_database *sdb) { if (!sdb->settings) { - yaz_log(YLOG_WARN, "No settings associated with %s", sdb->database->url); + yaz_log(YLOG_WARN, + "No settings associated with %s", sdb->database->url); return -1; } if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) @@ -435,7 +369,8 @@ static int prepare_session_database(struct session *se, struct session_database } -void session_set_watch(struct session *s, int what, session_watchfun fun, void *data) +void session_set_watch(struct session *s, int what, + session_watchfun fun, void *data) { s->watchlist[what].fun = fun; s->watchlist[what].data = data; @@ -460,7 +395,8 @@ static void select_targets_callback(void *context, struct session_database *db) } // Associates a set of clients with a session; -// Note: Session-databases represent databases with per-session setting overrides +// Note: Session-databases represent databases with per-session +// setting overrides int select_targets(struct session *se, struct database_criterion *crit) { while (se->clients) @@ -510,7 +446,8 @@ static struct database_criterion *parse_filter(NMEM m, const char *buf) new->values = 0; for (subi = 0; subi < subnum; subi++) { - struct database_criterion_value *newv = nmem_malloc(m, sizeof(*newv)); + struct database_criterion_value *newv + = nmem_malloc(m, sizeof(*newv)); newv->value = subvalues[subi]; newv->next = new->values; new->values = newv; @@ -551,7 +488,8 @@ char *search(struct session *se, char *query, char *filter) { if (prepare_session_database(se, client_get_database(cl)) < 0) return "CONFIG_ERROR"; - if (client_parse_query(cl, query) < 0) // Query must parse for all targets + // Query must parse for all targets + if (client_parse_query(cl, query) < 0) return "QUERY"; } @@ -574,8 +512,10 @@ static void session_init_databases_fun(void *context, struct database *db) new->database = db; new->yaz_marc = 0; new->map = 0; - new->settings = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); + new->settings + = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); memset(new->settings, 0, sizeof(struct settings*) * num); + if (db->settings) { for (i = 0; i < num; i++) @@ -606,7 +546,8 @@ void session_init_databases(struct session *se) // Probably session_init_databases_fun should be refactored instead of // called here. -static struct session_database *load_session_database(struct session *se, char *id) +static struct session_database *load_session_database(struct session *se, + char *id) { struct database *db = find_database(id, 0); @@ -616,7 +557,8 @@ static struct session_database *load_session_database(struct session *se, char * } // Find an existing session database. If not found, load it -static struct session_database *find_session_database(struct session *se, char *id) +static struct session_database *find_session_database(struct session *se, + char *id) { struct session_database *sdb; @@ -627,7 +569,8 @@ static struct session_database *find_session_database(struct session *se, char * } // Apply a session override to a database -void session_apply_setting(struct session *se, char *dbname, char *setting, char *value) +void session_apply_setting(struct session *se, char *dbname, char *setting, + char *value) { struct session_database *sdb = find_session_database(se, dbname); struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); @@ -766,8 +709,9 @@ struct record_cluster *show_single(struct session *s, int id) return 0; } -struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, int start, - int *num, int *total, int *sumhits, NMEM nmem_show) +struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, + int start, int *num, int *total, int *sumhits, + NMEM nmem_show) { struct record_cluster **recs = nmem_malloc(nmem_show, *num * sizeof(struct record_cluster *)); @@ -939,7 +883,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, { xmlDoc *xdoc = normalize_record(client_get_database(cl), rec); xmlNode *root, *n; - struct record *res; + struct record *record; struct record_cluster *cluster; struct session *se = client_get_session(cl); xmlChar *mergekey, *mergekey_norm; @@ -958,20 +902,28 @@ struct record *ingest_record(struct client *cl, Z_External *rec, return 0; } - res = nmem_malloc(se->nmem, sizeof(struct record)); - res->next = 0; - res->client = cl; - res->metadata = nmem_malloc(se->nmem, +#if 0 + record = nmem_malloc(se->nmem, sizeof(struct record)); + record->next = 0; + record->client = cl; + record->metadata = nmem_malloc(se->nmem, sizeof(struct record_metadata*) * service->num_metadata); - memset(res->metadata, 0, sizeof(struct record_metadata*) * service->num_metadata); + memset(record->metadata, 0, + sizeof(struct record_metadata*) * service->num_metadata); + +#else + record = record_create(se->nmem, + service->num_metadata, service->num_sortkeys); + record_assign_client(record, cl); +#endif mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); xmlFree(mergekey); - normalize_mergekey((char *) mergekey_norm, 0); + normalize7bit_mergekey((char *) mergekey_norm, 0); cluster = reclist_insert(se->reclist, global_parameters.server->service, - res, (char *) mergekey_norm, + record, (char *) mergekey_norm, &se->total_merged); if (global_parameters.dump_records) yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, @@ -996,10 +948,12 @@ struct record *ingest_record(struct client *cl, Z_External *rec, continue; if (!strcmp((const char *) n->name, "metadata")) { - struct conf_metadata *md = 0; - struct conf_sortkey *sk = 0; - struct record_metadata **wheretoput, *newm; - int imeta; + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; int first, last; type = xmlGetProp(n, (xmlChar *) "type"); @@ -1008,31 +962,62 @@ struct record *ingest_record(struct client *cl, Z_External *rec, if (!type || !value) continue; +#if 0 // First, find out what field we're looking at - for (imeta = 0; imeta < service->num_metadata; imeta++) - if (!strcmp((const char *) type, service->metadata[imeta].name)) + for (md_field_id = 0; md_field_id < service->num_metadata; + md_field_id++) + if (!strcmp((const char *) type, + service->metadata[md_field_id].name)) { - md = &service->metadata[imeta]; - if (md->sortkey_offset >= 0) - sk = &service->sortkeys[md->sortkey_offset]; + ser_md = &service->metadata[md_field_id]; + if (ser_md->sortkey_offset >= 0){ + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } break; } - if (!md) + + if (!ser_md) + { + yaz_log(YLOG_WARN, + "Ignoring unknown metadata element: %s", type); + continue; + } + +#else + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) { - yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type); + yaz_log(YLOG_WARN, + "Ignoring unknown metadata element: %s", type); continue; } + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0){ + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } +#endif + // Find out where we are putting it - if (md->merge == Metadata_merge_no) - wheretoput = &res->metadata[imeta]; + if (ser_md->merge == Metadata_merge_no) + wheretoput = &record->metadata[md_field_id]; else - wheretoput = &cluster->metadata[imeta]; + wheretoput = &cluster->metadata[md_field_id]; // Put it there - newm = nmem_malloc(se->nmem, sizeof(struct record_metadata)); - newm->next = 0; - if (md->type == Metadata_type_generic) + +#if 1 //something wrong with constructor, debug tomorrow + rec_md = nmem_malloc(se->nmem, sizeof(struct record_metadata)); + rec_md->next = 0; +#else + rec_md = record_metadata_create(se->nmem); +#endif + + if (ser_md->type == Metadata_type_generic) { char *p, *pe; for (p = (char *) value; *p && isspace(*p); p++) @@ -1040,69 +1025,78 @@ struct record *ingest_record(struct client *cl, Z_External *rec, for (pe = p + strlen(p) - 1; pe > p && strchr(" ,/.:([", *pe); pe--) *pe = '\0'; - newm->data.text = nmem_strdup(se->nmem, p); + //char * normalize7bit_generic(char* str, char* rm_chars); + + rec_md->data.text = nmem_strdup(se->nmem, p); } - else if (md->type == Metadata_type_year) + else if (ser_md->type == Metadata_type_year) { if (extract_years((char *) value, &first, &last) < 0) continue; } else { - yaz_log(YLOG_WARN, "Unknown type in metadata element %s", type); + yaz_log(YLOG_WARN, + "Unknown type in metadata element %s", type); continue; } - if (md->type == Metadata_type_year && md->merge != Metadata_merge_range) + if (ser_md->type == Metadata_type_year + && ser_md->merge != Metadata_merge_range) { yaz_log(YLOG_WARN, "Only range merging supported for years"); continue; } - if (md->merge == Metadata_merge_unique) + if (ser_md->merge == Metadata_merge_unique) { struct record_metadata *mnode; for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text, newm->data.text)) + if (!strcmp((const char *) mnode->data.text, + rec_md->data.text)) break; if (!mnode) { - newm->next = *wheretoput; - *wheretoput = newm; + rec_md->next = *wheretoput; + *wheretoput = rec_md; } } - else if (md->merge == Metadata_merge_longest) + else if (ser_md->merge == Metadata_merge_longest) { - if (!*wheretoput || - strlen(newm->data.text) > strlen((*wheretoput)->data.text)) + if (!*wheretoput + || strlen(rec_md->data.text) + > strlen((*wheretoput)->data.text)) { - *wheretoput = newm; - if (sk) + *wheretoput = rec_md; + if (ser_sk) { - char *s = nmem_strdup(se->nmem, newm->data.text); - if (!cluster->sortkeys[md->sortkey_offset]) - cluster->sortkeys[md->sortkey_offset] = - nmem_malloc(se->nmem, sizeof(union data_types)); - normalize_mergekey(s, - (sk->type == Metadata_sortkey_skiparticle)); - cluster->sortkeys[md->sortkey_offset]->text = s; + char *s = nmem_strdup(se->nmem, rec_md->data.text); + if (!cluster->sortkeys[sk_field_id]) + cluster->sortkeys[sk_field_id] = + nmem_malloc(se->nmem, + sizeof(union data_types)); + normalize7bit_mergekey(s, + (ser_sk->type == Metadata_sortkey_skiparticle)); + cluster->sortkeys[sk_field_id]->text = s; } } } - else if (md->merge == Metadata_merge_all || md->merge == Metadata_merge_no) + else if (ser_md->merge == Metadata_merge_all + || ser_md->merge == Metadata_merge_no) { - newm->next = *wheretoput; - *wheretoput = newm; + rec_md->next = *wheretoput; + *wheretoput = rec_md; } - else if (md->merge == Metadata_merge_range) + else if (ser_md->merge == Metadata_merge_range) { - assert(md->type == Metadata_type_year); + assert(ser_md->type == Metadata_type_year); if (!*wheretoput) { - *wheretoput = newm; + *wheretoput = rec_md; (*wheretoput)->data.number.min = first; (*wheretoput)->data.number.max = last; - if (sk) - cluster->sortkeys[md->sortkey_offset] = &newm->data; + if (ser_sk) + cluster->sortkeys[sk_field_id] + = &rec_md->data; } else { @@ -1112,22 +1106,26 @@ struct record *ingest_record(struct client *cl, Z_External *rec, (*wheretoput)->data.number.max = last; } #ifdef GAGA - if (sk) + if (ser_sk) { - union data_types *sdata = cluster->sortkeys[md->sortkey_offset]; - yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max); + union data_types *sdata + = cluster->sortkeys[sk_field_id]; + yaz_log(YLOG_LOG, "SK range: %d-%d", + sdata->number.min, sdata->number.max); } #endif } else - yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", md->name); + yaz_log(YLOG_WARN, + "Don't know how to merge on element name %s", + ser_md->name); - if (md->rank) + if (ser_md->rank) relevance_countwords(se->relevance, cluster, - (char *) value, md->rank); - if (md->termlist) + (char *) value, ser_md->rank); + if (ser_md->termlist) { - if (md->type == Metadata_type_year) + if (ser_md->type == Metadata_type_year) { char year[64]; sprintf(year, "%d", last); @@ -1146,7 +1144,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, type = value = 0; } else - yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name); + yaz_log(YLOG_WARN, + "Unexpected element %s in internal record", n->name); } if (type) xmlFree(type); @@ -1158,7 +1157,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, relevance_donerecord(se->relevance, cluster); se->total_records++; - return res; + return record; }