-/* $Id: logic.c,v 1.1 2007-04-16 09:03:25 adam Exp $
+/* $Id: logic.c,v 1.19 2007-04-23 08:15:22 marc Exp $
Copyright (c) 2006-2007, Index Data.
This file is part of Pazpar2.
02111-1307, USA.
*/
+// This file contains the primary business logic. Several parts of it should
+// Eventually be factored into separate modules.
+
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
static void ingest_records(struct client *cl, Z_Records *r);
void session_alert_watch(struct session *s, int what);
-IOCHAN channel_list = 0; // Master list of connections we're handling events to
-
static struct connection *connection_freelist = 0;
static struct client *client_freelist = 0;
0,
30,
"81",
- "Index Data PazPar2 (MasterKey)",
+ "Index Data PazPar2",
VERSION,
600, // 10 minutes
60,
&& 0 < strlen(cl->database->database->url))
{
#if YAZ_VERSIONL >= 0x020163
- const int *oid_proxy = yaz_string_to_oid(yaz_oid_std(),
- CLASS_USERINFO, OID_STR_PROXY);
yaz_oi_set_string_oid(&a->u.initRequest->otherInfo,
- global_parameters.odr_out, oid_proxy,
+ global_parameters.odr_out,
+ yaz_oid_userinfo_proxy,
1, cl->database->database->url);
#else
yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo,
odr_reset(global_parameters.odr_out);
}
+// Recursively traverse query structure to extract terms.
static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num)
{
+ char **words;
+ int numwords;
+ int i;
+
switch (n->kind)
{
case CCL_RPN_AND:
pull_terms(nmem, n->u.p[1], termlist, num);
break;
case CCL_RPN_TERM:
- termlist[(*num)++] = nmem_strdup(nmem, n->u.t.term);
+ nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords);
+ for (i = 0; i < numwords; i++)
+ termlist[(*num)++] = words[i];
break;
default: // NOOP
break;
struct session *se = cl->session;
struct session_database *sdb = cl->database;
Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
- int ndb, cerror, cpos;
+ int ndb;
char **databaselist;
Z_Query *zquery;
- struct ccl_rpn_node *cn;
int ssub = 0, lslb = 100000, mspn = 10;
char *recsyn = 0;
char *piggyback = 0;
yaz_log(YLOG_DEBUG, "Sending search to %s", cl->database->database->url);
- cn = ccl_find_str(sdb->database->ccl_map, se->query, &cerror, &cpos);
- if (!cn)
- return;
-
- if (!se->relevance)
- {
- // Initialize relevance structure with query terms
- char *p[512];
- extract_terms(se->nmem, cn, p);
- se->relevance = relevance_create(se->nmem, (const char **) p,
- se->expected_maxrecs);
- }
-
// constructing RPN query
a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
sizeof(Z_Query));
zquery->which = Z_Query_type_1;
- zquery->u.type_1 = ccl_rpn_query(global_parameters.odr_out, cn);
- ccl_rpn_delete(cn);
+ zquery->u.type_1 = p_query_rpn(global_parameters.odr_out, cl->pquery);
// converting to target encoding
if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
if (i == SESSION_MAX_TERMLISTS)
{
yaz_log(YLOG_FATAL, "Too many termlists");
- exit(1);
+ return;
}
+
s->termlists[i].name = nmem_strdup(s->nmem, type);
s->termlists[i].termlist = termlist_create(s->nmem, s->expected_maxrecs, 15);
s->num_termlists = i + 1;
static xmlDoc *normalize_record(struct client *cl, Z_External *rec)
{
struct database_retrievalmap *m;
- struct database *db = cl->database->database;
+ struct session_database *sdb = cl->database;
+ struct database *db = sdb->database;
xmlNode *res;
xmlDoc *rdoc;
// First normalize to XML
- if (db->yaz_marc)
+ if (sdb->yaz_marc)
{
char *buf;
int len;
if (rec->which != Z_External_octet)
{
yaz_log(YLOG_WARN, "Unexpected external branch, probably BER %s",
- cl->database->database->url);
+ db->url);
return 0;
}
buf = (char*) rec->u.octet_aligned->buf;
len = rec->u.octet_aligned->len;
- if (yaz_marc_read_iso2709(db->yaz_marc, buf, len) < 0)
+ if (yaz_marc_read_iso2709(sdb->yaz_marc, buf, len) < 0)
{
- yaz_log(YLOG_WARN, "Failed to decode MARC %s",
- cl->database->database->url);
+ yaz_log(YLOG_WARN, "Failed to decode MARC %s", db->url);
return 0;
}
- yaz_marc_write_using_libxml2(db->yaz_marc, 1);
- if (yaz_marc_write_xml(db->yaz_marc, &res,
+ yaz_marc_write_using_libxml2(sdb->yaz_marc, 1);
+ if (yaz_marc_write_xml(sdb->yaz_marc, &res,
"http://www.loc.gov/MARC21/slim", 0, 0) < 0)
{
yaz_log(YLOG_WARN, "Failed to encode as XML %s",
- cl->database->database->url);
+ db->url);
return 0;
}
rdoc = xmlNewDoc((xmlChar *) "1.0");
{
yaz_log(YLOG_FATAL,
"Unknown native_syntax in normalize_record from %s",
- cl->database->database->url);
- exit(1);
+ db->url);
+ return 0;
}
if (global_parameters.dump_records){
fprintf(stderr,
"Input Record (normalized) from %s\n----------------\n",
- cl->database->database->url);
+ db->url);
#if LIBXML_VERSION >= 20600
xmlDocFormatDump(stderr, rdoc, 1);
#else
#endif
}
- for (m = db->map; m; m = m->next){
+ for (m = sdb->map; m; m = m->next){
xmlDoc *new = 0;
#if 1
xmlFree(mergekey);
normalize_mergekey((char *) mergekey_norm, 0);
- cluster = reclist_insert(se->reclist, res, (char *) mergekey_norm,
+ cluster = reclist_insert(se->reclist,
+ global_parameters.server->service,
+ res, (char *) mergekey_norm,
&se->total_merged);
if (global_parameters.dump_records)
yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid,
}
}
-static void handler(IOCHAN i, int event)
+void connection_handler(IOCHAN i, int event)
{
struct connection *co = iochan_getdata(i);
struct client *cl = co->client;
if (cl->state == Client_Idle)
{
- if (cl->requestid != se->requestid && *se->query) {
+ if (cl->requestid != se->requestid && cl->pquery) {
send_search(i);
}
else if (cl->hits > 0 && cl->records < global_parameters.toget &&
static void connection_destroy(struct connection *co)
{
struct host *h = co->host;
- cs_close(co->link);
- iochan_destroy(co->iochan);
+
+ if (co->link)
+ {
+ cs_close(co->link);
+ iochan_destroy(co->iochan);
+ }
yaz_log(YLOG_DEBUG, "Connection destroy %s", co->host->hostport);
if (h->connections == co)
connection_freelist = co;
}
-// Creates a new connection for client, associated with the host of
-// client's database
-static struct connection *connection_create(struct client *cl)
+static int connection_connect(struct connection *con)
{
- struct connection *new;
- COMSTACK link;
- int res;
+ COMSTACK link = 0;
+ struct client *cl = con->client;
+ struct host *host = con->host;
void *addr;
+ int res;
+ assert(host->ipport);
+ assert(cl);
if (!(link = cs_create(tcpip_type, 0, PROTO_Z3950)))
- {
- yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack");
- exit(1);
- }
+ {
+ yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack");
+ return -1;
+ }
if (0 == strlen(global_parameters.zproxy_override)){
/* no Z39.50 proxy needed - direct connect */
yaz_log(YLOG_DEBUG, "Connection create %s", cl->database->database->url);
- if (!(addr = cs_straddr(link, cl->database->database->host->ipport)))
- {
- yaz_log(YLOG_WARN|YLOG_ERRNO,
- "Lookup of IP address %s failed",
- cl->database->database->host->ipport);
- return 0;
- }
-
+ if (!(addr = cs_straddr(link, host->ipport)))
+ {
+ yaz_log(YLOG_WARN|YLOG_ERRNO,
+ "Lookup of IP address %s failed", host->ipport);
+ return -1;
+ }
+
} else {
/* Z39.50 proxy connect */
yaz_log(YLOG_DEBUG, "Connection create %s proxy %s",
cl->database->database->url, global_parameters.zproxy_override);
-
+
if (!(addr = cs_straddr(link, global_parameters.zproxy_override)))
- {
- yaz_log(YLOG_WARN|YLOG_ERRNO,
- "Lookup of IP address %s failed",
- global_parameters.zproxy_override);
- return 0;
- }
+ {
+ yaz_log(YLOG_WARN|YLOG_ERRNO,
+ "Lookup of IP address %s failed",
+ global_parameters.zproxy_override);
+ return -1;
+ }
}
-
+
res = cs_connect(link, addr);
if (res < 0)
{
yaz_log(YLOG_WARN|YLOG_ERRNO, "cs_connect %s", cl->database->database->url);
- return 0;
+ return -1;
+ }
+ con->link = link;
+ con->state = Conn_Connecting;
+ con->iochan = iochan_create(cs_fileno(link), connection_handler, 0);
+ iochan_setdata(con->iochan, con);
+ pazpar2_add_channel(con->iochan);
+
+ /* this fragment is bad DRY: from client_prep_connection */
+ cl->state = Client_Connecting;
+ iochan_setflag(con->iochan, EVENT_OUTPUT);
+ return 0;
+}
+
+void connect_resolver_host(struct host *host)
+{
+ struct connection *con = host->connections;
+
+ while (con)
+ {
+ if (con->state == Conn_Resolving)
+ {
+ if (!host->ipport) /* unresolved */
+ {
+ connection_destroy(con);
+ /* start all over .. at some point it will be NULL */
+ con = host->connections;
+ }
+ else if (!con->client)
+ {
+ yaz_log(YLOG_WARN, "connect_unresolved_host : ophan client");
+ connection_destroy(con);
+ /* start all over .. at some point it will be NULL */
+ con = host->connections;
+ }
+ else
+ {
+ connection_connect(con);
+ con = con->next;
+ }
+ }
}
+}
+
+
+// Creates a new connection for client, associated with the host of
+// client's database
+static struct connection *connection_create(struct client *cl)
+{
+ struct connection *new;
+ struct host *host = cl->database->database->host;
if ((new = connection_freelist))
connection_freelist = new->next;
new->ibuf = 0;
new->ibufsize = 0;
}
- new->state = Conn_Connecting;
- new->host = cl->database->database->host;
+ new->host = host;
new->next = new->host->connections;
new->host->connections = new;
new->client = cl;
cl->connection = new;
- new->link = link;
-
- new->iochan = iochan_create(cs_fileno(link), handler, 0);
- iochan_setdata(new->iochan, new);
- new->iochan->next = channel_list;
- channel_list = new->iochan;
+ new->link = 0;
+ new->state = Conn_Resolving;
+ if (host->ipport)
+ connection_connect(new);
return new;
}
return 0;
}
+// Initialize YAZ Map structures for MARC-based targets
+static int prepare_yazmarc(struct session_database *sdb)
+{
+ char *s;
+
+ if (!sdb->settings)
+ {
+ yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url);
+ return -1;
+ }
+ if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) && !strncmp(s, "iso2709", 7))
+ {
+ char *encoding = "marc-8s", *e;
+ yaz_iconv_t cm;
+
+ // See if a native encoding is specified
+ if ((e = strchr(s, ';')))
+ encoding = e + 1;
+
+ sdb->yaz_marc = yaz_marc_create();
+ yaz_marc_subfield_str(sdb->yaz_marc, "\t");
+
+ cm = yaz_iconv_open("utf-8", encoding);
+ if (!cm)
+ {
+ yaz_log(YLOG_FATAL,
+ "Unable to map from %s to UTF-8 for target %s",
+ encoding, sdb->database->url);
+ return -1;
+ }
+ yaz_marc_iconv(sdb->yaz_marc, cm);
+ }
+ return 0;
+}
+
+// Prepare XSLT stylesheets for record normalization
+// Structures are allocated on the session_wide nmem to avoid having
+// to recompute this for every search. This would lead
+// to leaking if a single session was to repeatedly change the PZ_XSLT
+// setting. However, this is not a realistic use scenario.
+static int prepare_map(struct session *se, struct session_database *sdb)
+{
+ char *s;
+
+ if (!sdb->settings)
+ {
+ yaz_log(YLOG_WARN, "No settings on %s", sdb->database->url);
+ return -1;
+ }
+ if ((s = session_setting_oneval(sdb, PZ_XSLT)))
+ {
+ char **stylesheets;
+ struct database_retrievalmap **m = &sdb->map;
+ int num, i;
+
+ nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num);
+ for (i = 0; i < num; i++)
+ {
+ (*m) = nmem_malloc(se->session_nmem, sizeof(**m));
+ (*m)->next = 0;
+ if (!((*m)->stylesheet = conf_load_stylesheet(stylesheets[i])))
+ {
+ yaz_log(YLOG_FATAL, "Unable to load stylesheet: %s",
+ stylesheets[i]);
+ return -1;
+ }
+ m = &(*m)->next;
+ }
+ }
+ if (!sdb->map)
+ yaz_log(YLOG_WARN, "No Normalization stylesheet for target %s",
+ sdb->database->url);
+ return 0;
+}
+
+// This analyzes settings and recomputes any supporting data structures
+// if necessary.
+static int prepare_session_database(struct session *se, struct session_database *sdb)
+{
+ if (!sdb->settings)
+ {
+ yaz_log(YLOG_WARN, "No settings associated with %s", sdb->database->url);
+ return -1;
+ }
+ if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc)
+ {
+ if (prepare_yazmarc(sdb) < 0)
+ return -1;
+ }
+ if (sdb->settings[PZ_XSLT] && !sdb->map)
+ {
+ if (prepare_map(se, sdb) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+// Initialize CCL map for a target
+static CCL_bibset prepare_cclmap(struct client *cl)
+{
+ struct session_database *sdb = cl->database;
+ struct setting *s;
+ CCL_bibset res;
+
+ if (!sdb->settings)
+ return 0;
+ res = ccl_qual_mk();
+ for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
+ {
+ char *p = strchr(s->name + 3, ':');
+ if (!p)
+ {
+ yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
+ ccl_qual_rm(&res);
+ return 0;
+ }
+ p++;
+ ccl_qual_fitem(res, s->value, p);
+ }
+ return res;
+}
+
+// Parse the query given the settings specific to this client
+static int client_parse_query(struct client *cl, const char *query)
+{
+ struct session *se = cl->session;
+ struct ccl_rpn_node *cn;
+ int cerror, cpos;
+ CCL_bibset ccl_map = prepare_cclmap(cl);
+
+ if (!ccl_map)
+ return -1;
+ cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
+ ccl_qual_rm(&ccl_map);
+ if (!cn)
+ {
+ cl->state = Client_Error;
+ yaz_log(YLOG_WARN, "Failed to parse query for %s",
+ cl->database->database->url);
+ return -1;
+ }
+ wrbuf_rewind(se->wrbuf);
+ ccl_pquery(se->wrbuf, cn);
+ wrbuf_putc(se->wrbuf, '\0');
+ if (cl->pquery)
+ xfree(cl->pquery);
+ cl->pquery = xstrdup(wrbuf_buf(se->wrbuf));
+
+ if (!se->relevance)
+ {
+ // Initialize relevance structure with query terms
+ char *p[512];
+ extract_terms(se->nmem, cn, p);
+ se->relevance = relevance_create(se->nmem, (const char **) p,
+ se->expected_maxrecs);
+ }
+
+ ccl_rpn_delete(cn);
+ return 0;
+}
+
static struct client *client_create(void)
{
struct client *r;
}
else
r = xmalloc(sizeof(struct client));
+ r->pquery = 0;
r->database = 0;
r->connection = 0;
r->session = 0;
nmem_reset(se->nmem);
criteria = parse_filter(se->nmem, filter);
- strcpy(se->query, query);
se->requestid++;
- select_targets(se, criteria);
- for (cl = se->clients; cl; cl = cl->next)
- {
- if (client_prep_connection(cl))
- live_channels++;
- }
+ live_channels = select_targets(se, criteria);
if (live_channels)
{
int maxrecs = live_channels * global_parameters.toget;
se->num_termlists = 0;
se->reclist = reclist_create(se->nmem, maxrecs);
// This will be initialized in send_search()
- se->relevance = 0;
se->total_records = se->total_hits = se->total_merged = 0;
se->expected_maxrecs = maxrecs;
}
else
return "NOTARGETS";
- return 0;
-}
+ se->relevance = 0;
-// Apply a session override to a database
-void session_apply_setting(struct session *se, char *dbname, char *setting, char *value)
-{
- struct session_database *sdb;
+ for (cl = se->clients; cl; cl = cl->next)
+ {
+ if (prepare_session_database(se, cl->database) < 0)
+ return "CONFIG_ERROR";
+ if (client_parse_query(cl, query) < 0) // Query must parse for all targets
+ return "QUERY";
+ }
- for (sdb = se->databases; sdb; sdb = sdb->next)
- if (!strcmp(dbname, sdb->database->url))
- {
- struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new));
- int offset = settings_offset(setting);
+ for (cl = se->clients; cl; cl = cl->next)
+ {
+ client_prep_connection(cl);
+ }
- if (offset < 0)
- {
- yaz_log(YLOG_WARN, "Unknown setting %s", setting);
- return;
- }
- new->precedence = 0;
- new->target = dbname;
- new->name = setting;
- new->value = value;
- new->next = sdb->settings[offset];
- sdb->settings[offset] = new;
- break;
- }
- if (!sdb)
- yaz_log(YLOG_WARN, "Unknown database in setting override: %s", dbname);
+ return 0;
}
-void session_init_databases_fun(void *context, struct database *db)
+// Creates a new session_database object for a database
+static void session_init_databases_fun(void *context, struct database *db)
{
struct session *se = (struct session *) context;
struct session_database *new = nmem_malloc(se->session_nmem, sizeof(*new));
int i;
new->database = db;
+ new->yaz_marc = 0;
+ new->map = 0;
new->settings = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num);
- for (i = 0; i < num; i++)
- new->settings[i] = db->settings[i];
+ memset(new->settings, 0, sizeof(struct settings*) * num);
+ if (db->settings)
+ {
+ for (i = 0; i < num; i++)
+ new->settings[i] = db->settings[i];
+ }
new->next = se->databases;
se->databases = new;
}
+// Doesn't free memory associated with sdb -- nmem takes care of that
+static void session_database_destroy(struct session_database *sdb)
+{
+ struct database_retrievalmap *m;
+
+ for (m = sdb->map; m; m = m->next)
+ xsltFreeStylesheet(m->stylesheet);
+ if (sdb->yaz_marc)
+ yaz_marc_destroy(sdb->yaz_marc);
+}
+
// Initialize session_database list -- this represents this session's view
// of the database list -- subject to modification by the settings ws command
void session_init_databases(struct session *se)
grep_databases(se, 0, session_init_databases_fun);
}
+// Probably session_init_databases_fun should be refactored instead of
+// called here.
+static struct session_database *load_session_database(struct session *se, char *id)
+{
+ struct database *db = find_database(id, 0);
+
+ session_init_databases_fun((void*) se, db);
+ // New sdb is head of se->databases list
+ return se->databases;
+}
+
+// Find an existing session database. If not found, load it
+static struct session_database *find_session_database(struct session *se, char *id)
+{
+ struct session_database *sdb;
+
+ for (sdb = se->databases; sdb; sdb = sdb->next)
+ if (!strcmp(sdb->database->url, id))
+ return sdb;
+ return load_session_database(se, id);
+}
+
+// Apply a session override to a database
+void session_apply_setting(struct session *se, char *dbname, char *setting, char *value)
+{
+ struct session_database *sdb = find_session_database(se, dbname);
+ struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new));
+ int offset = settings_offset(setting);
+
+ if (offset < 0)
+ {
+ yaz_log(YLOG_WARN, "Unknown setting %s", setting);
+ return;
+ }
+ new->precedence = 0;
+ new->target = dbname;
+ new->name = setting;
+ new->value = value;
+ new->next = sdb->settings[offset];
+ sdb->settings[offset] = new;
+
+ // Force later recompute of settings-driven data structures
+ // (happens when a search starts and client connections are prepared)
+ switch (offset)
+ {
+ case PZ_NATIVESYNTAX:
+ if (sdb->yaz_marc)
+ {
+ yaz_marc_destroy(sdb->yaz_marc);
+ sdb->yaz_marc = 0;
+ }
+ break;
+ case PZ_XSLT:
+ if (sdb->map)
+ {
+ struct database_retrievalmap *m;
+ // We don't worry about the map structure -- it's in nmem
+ for (m = sdb->map; m; m = m->next)
+ xsltFreeStylesheet(m->stylesheet);
+ sdb->map = 0;
+ }
+ break;
+ }
+}
+
void destroy_session(struct session *s)
{
+ struct session_database *sdb;
+
yaz_log(YLOG_LOG, "Destroying session");
while (s->clients)
client_destroy(s->clients);
+ for (sdb = s->databases; sdb; sdb = sdb->next)
+ session_database_destroy(sdb);
nmem_destroy(s->nmem);
wrbuf_destroy(s->wrbuf);
}
session->requestid = -1;
session->clients = 0;
session->expected_maxrecs = 0;
- session->query[0] = '\0';
session->session_nmem = nmem;
session->nmem = nmem_create();
session->wrbuf = wrbuf_alloc();
return;
}
+// Master list of connections we're handling events to
+static IOCHAN channel_list = 0;
+void pazpar2_add_channel(IOCHAN chan)
+{
+ chan->next = channel_list;
+ channel_list = chan;
+}
+void pazpar2_event_loop()
+{
+ event_loop(&channel_list);
+}
/*
* Local variables:
projects / pazpar2-moved-to-github.git / blobdiff