1 /* $Id: client.c,v 1.9 2007-06-15 06:45:39 adam Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
31 #include <sys/socket.h>
37 #include <yaz/marcdisp.h>
38 #include <yaz/comstack.h>
39 #include <yaz/tcpip.h>
40 #include <yaz/proto.h>
41 #include <yaz/readconf.h>
42 #include <yaz/pquery.h>
43 #include <yaz/otherinfo.h>
44 #include <yaz/yaz-util.h>
46 #include <yaz/query-charset.h>
47 #include <yaz/querytowrbuf.h>
48 #include <yaz/oid_db.h>
56 #include <yaz/timing.h>
59 #include <netinet/in.h>
64 #include "connection.h"
67 /** \brief Represents client state for a connection to one search target */
69 struct session_database *database;
70 struct connection *connection;
71 struct session *session;
72 char *pquery; // Current search
76 int requestid; // ID of current outstanding request
78 enum client_state state;
79 struct show_raw *show_raw;
80 struct client *next; // next client in session or next in free list
84 int active; // whether this request has been sent to the server
88 void (*error_handler)(void *data, const char *addinfo);
89 void (*record_handler)(void *data, const char *buf, size_t sz);
93 static const char *client_states[] = {
97 "Client_Initializing",
102 "Client_Disconnected",
106 static struct client *client_freelist = 0;
108 static int send_apdu(struct client *c, Z_APDU *a)
110 return connection_send_apdu(client_get_connection(c), a);
114 const char *client_get_state_str(struct client *cl)
116 return client_states[cl->state];
119 enum client_state client_get_state(struct client *cl)
124 void client_set_state(struct client *cl, enum client_state st)
129 static void client_show_raw_error(struct client *cl, const char *addinfo);
131 // Close connection and set state to error
132 void client_fatal(struct client *cl)
134 client_show_raw_error(cl, "client connection failure");
135 yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
136 connection_destroy(cl->connection);
137 cl->state = Client_Error;
140 struct connection *client_get_connection(struct client *cl)
142 return cl->connection;
145 struct session_database *client_get_database(struct client *cl)
150 struct session *client_get_session(struct client *cl)
155 const char *client_get_pquery(struct client *cl)
160 void client_set_requestid(struct client *cl, int id)
165 int client_show_raw(struct client *cl, int position,
166 const char *syntax, const char *esn,
168 void (*error_handler)(void *data, const char *addinfo),
169 void (*record_handler)(void *data, const char *buf,
174 cl->show_raw = xmalloc(sizeof(*cl->show_raw));
175 cl->show_raw->position = position;
176 cl->show_raw->active = 0;
177 cl->show_raw->data = data;
178 cl->show_raw->error_handler = error_handler;
179 cl->show_raw->record_handler = record_handler;
181 cl->show_raw->syntax = xstrdup(syntax);
183 cl->show_raw->syntax = 0;
185 cl->show_raw->esn = xstrdup(esn);
187 cl->show_raw->esn = 0;
192 static void client_show_raw_error(struct client *cl, const char *addinfo)
196 cl->show_raw->error_handler(cl->show_raw->data, addinfo);
202 static void client_show_raw_cancel(struct client *cl)
206 cl->show_raw->error_handler(cl->show_raw->data, "cancel");
212 void client_send_raw_present(struct client *cl)
214 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
216 int start = cl->show_raw->position;
218 assert(cl->show_raw);
220 yaz_log(YLOG_LOG, "Trying to present %d record(s) from %d",
223 a->u.presentRequest->resultSetStartPoint = &start;
224 a->u.presentRequest->numberOfRecordsRequested = &toget;
226 if (cl->show_raw->syntax) // syntax is optional
227 a->u.presentRequest->preferredRecordSyntax =
228 yaz_string_to_oid_odr(yaz_oid_std(),
229 CLASS_RECSYN, cl->show_raw->syntax,
230 global_parameters.odr_out);
231 if (cl->show_raw->esn) // element set is optional
233 Z_ElementSetNames *elementSetNames =
234 odr_malloc(global_parameters.odr_out, sizeof(*elementSetNames));
235 Z_RecordComposition *compo =
236 odr_malloc(global_parameters.odr_out, sizeof(*compo));
237 a->u.presentRequest->recordComposition = compo;
239 compo->which = Z_RecordComp_simple;
240 compo->u.simple = elementSetNames;
242 elementSetNames->which = Z_ElementSetNames_generic;
243 elementSetNames->u.generic =
244 odr_strdup(global_parameters.odr_out, cl->show_raw->esn);
246 if (send_apdu(cl, a) >= 0)
248 cl->show_raw->active = 1;
249 cl->state = Client_Presenting;
253 client_show_raw_error(cl, "send_apdu failed");
254 cl->state = Client_Error;
256 odr_reset(global_parameters.odr_out);
259 void client_send_present(struct client *cl)
261 struct session_database *sdb = client_get_database(cl);
262 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
264 int start = cl->records + 1;
267 toget = global_parameters.chunk;
268 if (toget > global_parameters.toget - cl->records)
269 toget = global_parameters.toget - cl->records;
270 if (toget > cl->hits - cl->records)
271 toget = cl->hits - cl->records;
273 yaz_log(YLOG_DEBUG, "Trying to present %d record(s) from %d",
276 a->u.presentRequest->resultSetStartPoint = &start;
277 a->u.presentRequest->numberOfRecordsRequested = &toget;
279 if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
281 a->u.presentRequest->preferredRecordSyntax =
282 yaz_string_to_oid_odr(yaz_oid_std(),
283 CLASS_RECSYN, recsyn,
284 global_parameters.odr_out);
287 if (send_apdu(cl, a) >= 0)
288 cl->state = Client_Presenting;
290 cl->state = Client_Error;
291 odr_reset(global_parameters.odr_out);
295 void client_send_search(struct client *cl)
297 struct session *se = client_get_session(cl);
298 struct session_database *sdb = client_get_database(cl);
299 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
303 int ssub = 0, lslb = 100000, mspn = 10;
307 yaz_iconv_t iconv = 0;
309 yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
312 // constructing RPN query
313 a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
315 zquery->which = Z_Query_type_1;
316 zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
317 client_get_pquery(cl));
319 // converting to target encoding
320 if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
321 iconv = yaz_iconv_open(queryenc, "UTF-8");
323 yaz_query_charset_convert_rpnquery(zquery->u.type_1,
324 global_parameters.odr_out,
326 yaz_iconv_close(iconv);
328 yaz_log(YLOG_WARN, "Query encoding failed %s %s",
329 client_get_database(cl)->database->url, queryenc);
332 for (ndb = 0; sdb->database->databases[ndb]; ndb++)
334 databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
335 for (ndb = 0; sdb->database->databases[ndb]; ndb++)
336 databaselist[ndb] = sdb->database->databases[ndb];
338 if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK))
339 || *piggyback == '1')
341 if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
343 a->u.searchRequest->preferredRecordSyntax =
344 yaz_string_to_oid_odr(yaz_oid_std(),
345 CLASS_RECSYN, recsyn,
346 global_parameters.odr_out);
348 a->u.searchRequest->smallSetUpperBound = &ssub;
349 a->u.searchRequest->largeSetLowerBound = &lslb;
350 a->u.searchRequest->mediumSetPresentNumber = &mspn;
352 a->u.searchRequest->resultSetName = "Default";
353 a->u.searchRequest->databaseNames = databaselist;
354 a->u.searchRequest->num_databaseNames = ndb;
357 { //scope for sending and logging queries
358 WRBUF wbquery = wrbuf_alloc();
359 yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
362 if (send_apdu(cl, a) >= 0)
364 client_set_state(cl, Client_Searching);
365 client_set_requestid(cl, se->requestid);
366 yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
367 client_get_database(cl)->database->url,
368 queryenc ? queryenc : "UTF-8",
369 wrbuf_cstr(wbquery));
372 client_set_state(cl, Client_Error);
373 yaz_log(YLOG_WARN, "Failed SearchRequest %s %s %s",
374 client_get_database(cl)->database->url,
375 queryenc ? queryenc : "UTF-8",
376 wrbuf_cstr(wbquery));
379 wrbuf_destroy(wbquery);
382 odr_reset(global_parameters.odr_out);
385 void client_init_response(struct client *cl, Z_APDU *a)
387 Z_InitResponse *r = a->u.initResponse;
389 yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
393 cl->state = Client_Idle;
396 cl->state = Client_Failed; // FIXME need to do something to the connection
400 static void ingest_raw_records(struct client *cl, Z_Records *r)
402 Z_NamePlusRecordList *rlist;
403 Z_NamePlusRecord *npr;
407 if (r->which != Z_Records_DBOSD)
409 client_show_raw_error(cl, "non-surrogate diagnostics");
413 rlist = r->u.databaseOrSurDiagnostics;
414 if (rlist->num_records != 1 || !rlist->records || !rlist->records[0])
416 client_show_raw_error(cl, "no records");
419 npr = rlist->records[0];
420 if (npr->which != Z_NamePlusRecord_databaseRecord)
422 client_show_raw_error(cl, "surrogate diagnostic");
426 doc = record_to_xml(client_get_database(cl), npr->u.databaseRecord);
429 client_show_raw_error(cl, "unable to convert record to xml");
433 xmlDocDumpMemory(doc, &buf_out, &len_out);
435 cl->show_raw->record_handler(cl->show_raw->data,
436 (const char *) buf_out, len_out);
443 static void ingest_records(struct client *cl, Z_Records *r)
446 yaz_timing_t t = yaz_timing_create();
449 struct session *s = client_get_session(cl);
450 Z_NamePlusRecordList *rlist;
453 if (r->which != Z_Records_DBOSD)
455 rlist = r->u.databaseOrSurDiagnostics;
456 for (i = 0; i < rlist->num_records; i++)
458 Z_NamePlusRecord *npr = rlist->records[i];
461 if (npr->which != Z_NamePlusRecord_databaseRecord)
464 "Unexpected record type, probably diagnostic %s",
465 cl->database->database->url);
469 rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
473 if (rlist->num_records)
474 session_alert_watch(s, SESSION_WATCH_RECORDS);
478 yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
479 yaz_timing_get_real(t), yaz_timing_get_user(t),
480 yaz_timing_get_sys(t));
481 yaz_timing_destroy(&t);
486 void client_search_response(struct client *cl, Z_APDU *a)
488 struct session *se = cl->session;
489 Z_SearchResponse *r = a->u.searchResponse;
491 yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
492 cl->database->database->url, *r->searchStatus);
494 if (*r->searchStatus)
496 cl->hits = *r->resultCount;
497 se->total_hits += cl->hits;
498 if (r->presentStatus && !*r->presentStatus && r->records)
500 yaz_log(YLOG_DEBUG, "Records in search response %s",
501 cl->database->database->url);
502 ingest_records(cl, r->records);
504 cl->state = Client_Idle;
509 cl->state = Client_Error;
511 Z_Records *recs = r->records;
512 if (recs->which == Z_Records_NSD)
515 "Search response: Non-surrogate diagnostic %s (%d)",
516 cl->database->database->url,
517 *recs->u.nonSurrogateDiagnostic->condition);
518 cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
519 cl->state = Client_Error;
525 void client_present_response(struct client *cl, Z_APDU *a)
527 Z_PresentResponse *r = a->u.presentResponse;
530 Z_Records *recs = r->records;
531 if (recs->which == Z_Records_NSD)
533 yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s",
534 cl->database->database->url);
535 cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
536 cl->state = Client_Error;
537 client_show_raw_error(cl, "non surrogate diagnostics");
541 if (!*r->presentStatus && cl->state != Client_Error)
543 yaz_log(YLOG_DEBUG, "Good Present response %s",
544 cl->database->database->url);
546 // we can mix show raw and normal show ..
547 if (cl->show_raw && cl->show_raw->active)
549 cl->show_raw->active = 0; // no longer active
550 ingest_raw_records(cl, r->records);
553 ingest_records(cl, r->records);
554 cl->state = Client_Idle;
556 else if (*r->presentStatus)
558 yaz_log(YLOG_WARN, "Bad Present response %s",
559 cl->database->database->url);
560 cl->state = Client_Error;
561 client_show_raw_error(cl, "bad present response");
565 void client_close_response(struct client *cl, Z_APDU *a)
567 struct connection *co = cl->connection;
568 /* Z_Close *r = a->u.close; */
570 yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
572 cl->state = Client_Failed;
573 connection_destroy(co);
576 int client_is_our_response(struct client *cl)
578 struct session *se = client_get_session(cl);
580 if (cl && (cl->requestid == se->requestid ||
581 cl->state == Client_Initializing))
586 // Set authentication token in init if one is set for the client
587 // TODO: Extend this to handle other schemes than open (should be simple)
588 static void init_authentication(struct client *cl, Z_InitRequest *req)
590 struct session_database *sdb = client_get_database(cl);
591 char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
595 struct connection *co = client_get_connection(cl);
596 struct session *se = client_get_session(cl);
597 Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
599 idAuth->which = Z_IdAuthentication_open;
600 idAuth->u.open = auth;
601 req->idAuthentication = idAuth;
602 connection_set_authentication(co, nmem_strdup(se->session_nmem, auth));
606 static void init_zproxy(struct client *cl, Z_InitRequest *req)
608 struct session_database *sdb = client_get_database(cl);
609 char *ztarget = sdb->database->url;
610 //char *ztarget = sdb->url;
611 char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY);
614 yaz_oi_set_string_oid(&req->otherInfo,
615 global_parameters.odr_out,
616 yaz_oid_userinfo_proxy,
621 static void client_init_request(struct client *cl)
623 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
625 a->u.initRequest->implementationId = global_parameters.implementationId;
626 a->u.initRequest->implementationName = global_parameters.implementationName;
627 a->u.initRequest->implementationVersion =
628 global_parameters.implementationVersion;
629 ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
630 ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
631 ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
633 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
634 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
635 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
637 init_authentication(cl, a->u.initRequest);
638 init_zproxy(cl, a->u.initRequest);
640 if (send_apdu(cl, a) >= 0)
641 client_set_state(cl, Client_Initializing);
643 client_set_state(cl, Client_Error);
644 odr_reset(global_parameters.odr_out);
647 void client_continue(struct client *cl)
649 if (cl->state == Client_Connected) {
650 client_init_request(cl);
653 if (cl->state == Client_Idle)
655 struct session *se = client_get_session(cl);
656 if (cl->requestid != se->requestid && cl->pquery) {
657 // we'll have to abort this because result set is to be deleted
658 client_show_raw_cancel(cl);
659 client_send_search(cl);
661 else if (cl->show_raw)
663 client_send_raw_present(cl);
665 else if (cl->hits > 0 && cl->records < global_parameters.toget &&
666 cl->records < cl->hits) {
667 client_send_present(cl);
672 struct client *client_create(void)
678 client_freelist = client_freelist->next;
681 r = xmalloc(sizeof(struct client));
691 r->state = Client_Disconnected;
697 void client_destroy(struct client *c)
699 struct session *se = c->session;
700 if (c == se->clients)
701 se->clients = c->next;
705 for (cc = se->clients; cc && cc->next != c; cc = cc->next)
711 connection_release(c->connection);
712 c->next = client_freelist;
716 void client_set_connection(struct client *cl, struct connection *con)
718 cl->connection = con;
721 void client_disconnect(struct client *cl)
723 if (cl->state != Client_Idle)
724 cl->state = Client_Disconnected;
725 client_set_connection(cl, 0);
728 // Extract terms from query into null-terminated termlist
729 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
733 pull_terms(nmem, query, termlist, &num);
737 // Initialize CCL map for a target
738 static CCL_bibset prepare_cclmap(struct client *cl)
740 struct session_database *sdb = client_get_database(cl);
747 for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
749 char *p = strchr(s->name + 3, ':');
752 yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
757 ccl_qual_fitem(res, s->value, p);
762 // Parse the query given the settings specific to this client
763 int client_parse_query(struct client *cl, const char *query)
765 struct session *se = client_get_session(cl);
766 struct ccl_rpn_node *cn;
768 CCL_bibset ccl_map = prepare_cclmap(cl);
772 cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
773 ccl_qual_rm(&ccl_map);
776 cl->state = Client_Error;
777 yaz_log(YLOG_WARN, "Failed to parse query for %s",
778 client_get_database(cl)->database->url);
781 wrbuf_rewind(se->wrbuf);
782 ccl_pquery(se->wrbuf, cn);
784 cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
788 // Initialize relevance structure with query terms
790 extract_terms(se->nmem, cn, p);
791 se->relevance = relevance_create(client_get_database(cl)->pct,
792 se->nmem, (const char **) p,
793 se->expected_maxrecs);
800 void client_set_session(struct client *cl, struct session *se)
803 cl->next = se->clients;
807 int client_is_active(struct client *cl)
809 if (cl->connection && (cl->state == Client_Connecting ||
810 cl->state == Client_Initializing ||
811 cl->state == Client_Searching ||
812 cl->state == Client_Presenting))
817 struct client *client_next_in_session(struct client *cl)
825 int client_get_hits(struct client *cl)
830 int client_get_num_records(struct client *cl)
835 int client_get_diagnostic(struct client *cl)
837 return cl->diagnostic;
840 void client_set_database(struct client *cl, struct session_database *db)
845 struct host *client_get_host(struct client *cl)
847 return client_get_database(cl)->database->host;
850 const char *client_get_url(struct client *cl)
852 return client_get_database(cl)->database->url;
858 * indent-tabs-mode: nil
860 * vim: shiftwidth=4 tabstop=8 expandtab