1 /* $Id: client.c,v 1.12 2007-06-19 12:25:29 adam Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
31 #include <sys/socket.h>
37 #include <yaz/marcdisp.h>
38 #include <yaz/comstack.h>
39 #include <yaz/tcpip.h>
40 #include <yaz/proto.h>
41 #include <yaz/readconf.h>
42 #include <yaz/pquery.h>
43 #include <yaz/otherinfo.h>
44 #include <yaz/yaz-util.h>
46 #include <yaz/query-charset.h>
47 #include <yaz/querytowrbuf.h>
48 #include <yaz/oid_db.h>
49 #include <yaz/diagbib1.h>
57 #include <yaz/timing.h>
60 #include <netinet/in.h>
65 #include "connection.h"
68 /** \brief Represents client state for a connection to one search target */
70 struct session_database *database;
71 struct connection *connection;
72 struct session *session;
73 char *pquery; // Current search
77 int requestid; // ID of current outstanding request
79 enum client_state state;
80 struct show_raw *show_raw;
81 struct client *next; // next client in session or next in free list
85 int active; // whether this request has been sent to the server
89 void (*error_handler)(void *data, const char *addinfo);
90 void (*record_handler)(void *data, const char *buf, size_t sz);
94 static const char *client_states[] = {
98 "Client_Initializing",
103 "Client_Disconnected",
107 static struct client *client_freelist = 0;
109 static int send_apdu(struct client *c, Z_APDU *a)
111 return connection_send_apdu(client_get_connection(c), a);
115 const char *client_get_state_str(struct client *cl)
117 return client_states[cl->state];
120 enum client_state client_get_state(struct client *cl)
125 void client_set_state(struct client *cl, enum client_state st)
130 static void client_show_raw_error(struct client *cl, const char *addinfo);
132 // Close connection and set state to error
133 void client_fatal(struct client *cl)
135 client_show_raw_error(cl, "client connection failure");
136 yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
137 connection_destroy(cl->connection);
138 cl->state = Client_Error;
142 static int diag_to_wrbuf(Z_DiagRec **pp, int num, WRBUF w)
146 for (i = 0; i<num; i++)
148 Z_DiagRec *p = pp[i];
151 if (p->which != Z_DiagRec_defaultFormat)
153 wrbuf_puts(w, "? Not in default format");
157 Z_DefaultDiagFormat *r = p->u.defaultFormat;
159 if (!r->diagnosticSetId)
160 wrbuf_puts(w, "? Missing diagset");
164 char diag_name_buf[OID_STR_MAX];
165 const char *diag_name = 0;
166 diag_name = yaz_oid_to_string_buf
167 (r->diagnosticSetId, &oclass, diag_name_buf);
168 wrbuf_puts(w, diag_name);
171 code = *r->condition;
172 wrbuf_printf(w, " %d %s", *r->condition,
173 diagbib1_str(*r->condition));
176 case Z_DefaultDiagFormat_v2Addinfo:
177 wrbuf_printf(w, " -- v2 addinfo '%s'", r->u.v2Addinfo);
179 case Z_DefaultDiagFormat_v3Addinfo:
180 wrbuf_printf(w, " -- v3 addinfo '%s'", r->u.v3Addinfo);
190 struct connection *client_get_connection(struct client *cl)
192 return cl->connection;
195 struct session_database *client_get_database(struct client *cl)
200 struct session *client_get_session(struct client *cl)
205 const char *client_get_pquery(struct client *cl)
210 void client_set_requestid(struct client *cl, int id)
215 int client_show_raw_begin(struct client *cl, int position,
216 const char *syntax, const char *esn,
218 void (*error_handler)(void *data, const char *addinfo),
219 void (*record_handler)(void *data, const char *buf,
224 cl->show_raw = xmalloc(sizeof(*cl->show_raw));
225 cl->show_raw->position = position;
226 cl->show_raw->active = 0;
227 cl->show_raw->data = data;
228 cl->show_raw->error_handler = error_handler;
229 cl->show_raw->record_handler = record_handler;
231 cl->show_raw->syntax = xstrdup(syntax);
233 cl->show_raw->syntax = 0;
235 cl->show_raw->esn = xstrdup(esn);
237 cl->show_raw->esn = 0;
242 void client_show_raw_reset(struct client *cl)
248 static void client_show_raw_error(struct client *cl, const char *addinfo)
252 cl->show_raw->error_handler(cl->show_raw->data, addinfo);
253 client_show_raw_reset(cl);
257 static void client_show_raw_cancel(struct client *cl)
261 cl->show_raw->error_handler(cl->show_raw->data, "cancel");
262 client_show_raw_reset(cl);
266 void client_send_raw_present(struct client *cl)
268 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
270 int start = cl->show_raw->position;
272 assert(cl->show_raw);
274 yaz_log(YLOG_DEBUG, "Trying to present %d record(s) from %d",
277 a->u.presentRequest->resultSetStartPoint = &start;
278 a->u.presentRequest->numberOfRecordsRequested = &toget;
280 if (cl->show_raw->syntax) // syntax is optional
281 a->u.presentRequest->preferredRecordSyntax =
282 yaz_string_to_oid_odr(yaz_oid_std(),
283 CLASS_RECSYN, cl->show_raw->syntax,
284 global_parameters.odr_out);
285 if (cl->show_raw->esn) // element set is optional
287 Z_ElementSetNames *elementSetNames =
288 odr_malloc(global_parameters.odr_out, sizeof(*elementSetNames));
289 Z_RecordComposition *compo =
290 odr_malloc(global_parameters.odr_out, sizeof(*compo));
291 a->u.presentRequest->recordComposition = compo;
293 compo->which = Z_RecordComp_simple;
294 compo->u.simple = elementSetNames;
296 elementSetNames->which = Z_ElementSetNames_generic;
297 elementSetNames->u.generic =
298 odr_strdup(global_parameters.odr_out, cl->show_raw->esn);
300 if (send_apdu(cl, a) >= 0)
302 cl->show_raw->active = 1;
303 cl->state = Client_Presenting;
307 client_show_raw_error(cl, "send_apdu failed");
308 cl->state = Client_Error;
310 odr_reset(global_parameters.odr_out);
313 void client_send_present(struct client *cl)
315 struct session_database *sdb = client_get_database(cl);
316 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
318 int start = cl->records + 1;
321 toget = global_parameters.chunk;
322 if (toget > global_parameters.toget - cl->records)
323 toget = global_parameters.toget - cl->records;
324 if (toget > cl->hits - cl->records)
325 toget = cl->hits - cl->records;
327 yaz_log(YLOG_DEBUG, "Trying to present %d record(s) from %d",
330 a->u.presentRequest->resultSetStartPoint = &start;
331 a->u.presentRequest->numberOfRecordsRequested = &toget;
333 if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
335 a->u.presentRequest->preferredRecordSyntax =
336 yaz_string_to_oid_odr(yaz_oid_std(),
337 CLASS_RECSYN, recsyn,
338 global_parameters.odr_out);
341 if (send_apdu(cl, a) >= 0)
342 cl->state = Client_Presenting;
344 cl->state = Client_Error;
345 odr_reset(global_parameters.odr_out);
349 void client_send_search(struct client *cl)
351 struct session *se = client_get_session(cl);
352 struct session_database *sdb = client_get_database(cl);
353 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
357 int ssub = 0, lslb = 100000, mspn = 10;
361 yaz_iconv_t iconv = 0;
363 yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
366 // constructing RPN query
367 a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
369 zquery->which = Z_Query_type_1;
370 zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
371 client_get_pquery(cl));
373 // converting to target encoding
374 if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
375 iconv = yaz_iconv_open(queryenc, "UTF-8");
377 yaz_query_charset_convert_rpnquery(zquery->u.type_1,
378 global_parameters.odr_out,
380 yaz_iconv_close(iconv);
382 yaz_log(YLOG_WARN, "Query encoding failed %s %s",
383 client_get_database(cl)->database->url, queryenc);
386 for (ndb = 0; sdb->database->databases[ndb]; ndb++)
388 databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
389 for (ndb = 0; sdb->database->databases[ndb]; ndb++)
390 databaselist[ndb] = sdb->database->databases[ndb];
392 if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK))
393 || *piggyback == '1')
395 if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
397 a->u.searchRequest->preferredRecordSyntax =
398 yaz_string_to_oid_odr(yaz_oid_std(),
399 CLASS_RECSYN, recsyn,
400 global_parameters.odr_out);
402 a->u.searchRequest->smallSetUpperBound = &ssub;
403 a->u.searchRequest->largeSetLowerBound = &lslb;
404 a->u.searchRequest->mediumSetPresentNumber = &mspn;
406 a->u.searchRequest->databaseNames = databaselist;
407 a->u.searchRequest->num_databaseNames = ndb;
410 { //scope for sending and logging queries
411 WRBUF wbquery = wrbuf_alloc();
412 yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
415 if (send_apdu(cl, a) >= 0)
417 client_set_state(cl, Client_Searching);
418 client_set_requestid(cl, se->requestid);
419 yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
420 client_get_database(cl)->database->url,
421 queryenc ? queryenc : "UTF-8",
422 wrbuf_cstr(wbquery));
425 client_set_state(cl, Client_Error);
426 yaz_log(YLOG_WARN, "Failed SearchRequest %s %s %s",
427 client_get_database(cl)->database->url,
428 queryenc ? queryenc : "UTF-8",
429 wrbuf_cstr(wbquery));
432 wrbuf_destroy(wbquery);
435 odr_reset(global_parameters.odr_out);
438 void client_init_response(struct client *cl, Z_APDU *a)
440 Z_InitResponse *r = a->u.initResponse;
442 yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
446 cl->state = Client_Idle;
449 cl->state = Client_Failed; // FIXME need to do something to the connection
453 static void ingest_raw_records(struct client *cl, Z_Records *r)
455 Z_NamePlusRecordList *rlist;
456 Z_NamePlusRecord *npr;
460 if (r->which != Z_Records_DBOSD)
462 client_show_raw_error(cl, "non-surrogate diagnostics");
466 rlist = r->u.databaseOrSurDiagnostics;
467 if (rlist->num_records != 1 || !rlist->records || !rlist->records[0])
469 client_show_raw_error(cl, "no records");
472 npr = rlist->records[0];
473 if (npr->which != Z_NamePlusRecord_databaseRecord)
475 client_show_raw_error(cl, "surrogate diagnostic");
479 doc = record_to_xml(client_get_database(cl), npr->u.databaseRecord);
482 client_show_raw_error(cl, "unable to convert record to xml");
486 xmlDocDumpMemory(doc, &buf_out, &len_out);
489 cl->show_raw->record_handler(cl->show_raw->data,
490 (const char *) buf_out, len_out);
497 static void ingest_records(struct client *cl, Z_Records *r)
500 yaz_timing_t t = yaz_timing_create();
503 struct session *s = client_get_session(cl);
504 Z_NamePlusRecordList *rlist;
507 if (r->which != Z_Records_DBOSD)
509 rlist = r->u.databaseOrSurDiagnostics;
510 for (i = 0; i < rlist->num_records; i++)
512 Z_NamePlusRecord *npr = rlist->records[i];
515 if (npr->which != Z_NamePlusRecord_databaseRecord)
518 "Unexpected record type, probably diagnostic %s",
519 cl->database->database->url);
523 rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
527 if (rlist->num_records)
528 session_alert_watch(s, SESSION_WATCH_RECORDS);
532 yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
533 yaz_timing_get_real(t), yaz_timing_get_user(t),
534 yaz_timing_get_sys(t));
535 yaz_timing_destroy(&t);
540 void client_search_response(struct client *cl, Z_APDU *a)
542 struct session *se = cl->session;
543 Z_SearchResponse *r = a->u.searchResponse;
545 yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
546 cl->database->database->url, *r->searchStatus);
548 if (*r->searchStatus)
550 cl->hits = *r->resultCount;
551 se->total_hits += cl->hits;
552 if (r->presentStatus && !*r->presentStatus && r->records)
554 yaz_log(YLOG_DEBUG, "Records in search response %s",
555 cl->database->database->url);
556 ingest_records(cl, r->records);
558 cl->state = Client_Idle;
562 Z_Records *recs = r->records;
564 cl->state = Client_Error;
565 if (recs && recs->which == Z_Records_NSD)
567 WRBUF w = wrbuf_alloc();
569 Z_DiagRec dr, *dr_p = &dr;
570 dr.which = Z_DiagRec_defaultFormat;
571 dr.u.defaultFormat = recs->u.nonSurrogateDiagnostic;
573 wrbuf_printf(w, "Search response NSD %s: ",
574 cl->database->database->url);
576 cl->diagnostic = diag_to_wrbuf(&dr_p, 1, w);
578 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
580 cl->state = Client_Error;
583 else if (recs && recs->which == Z_Records_multipleNSD)
585 WRBUF w = wrbuf_alloc();
587 wrbuf_printf(w, "Search response multipleNSD %s: ",
588 cl->database->database->url);
590 diag_to_wrbuf(recs->u.multipleNonSurDiagnostics->diagRecs,
591 recs->u.multipleNonSurDiagnostics->num_diagRecs,
593 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
594 cl->state = Client_Error;
600 void client_present_response(struct client *cl, Z_APDU *a)
602 Z_PresentResponse *r = a->u.presentResponse;
603 Z_Records *recs = r->records;
605 if (recs && recs->which == Z_Records_NSD)
607 WRBUF w = wrbuf_alloc();
609 Z_DiagRec dr, *dr_p = &dr;
610 dr.which = Z_DiagRec_defaultFormat;
611 dr.u.defaultFormat = recs->u.nonSurrogateDiagnostic;
613 wrbuf_printf(w, "Present response NSD %s: ",
614 cl->database->database->url);
616 cl->diagnostic = diag_to_wrbuf(&dr_p, 1, w);
618 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
620 cl->state = Client_Error;
623 client_show_raw_error(cl, "non surrogate diagnostics");
625 else if (recs && recs->which == Z_Records_multipleNSD)
627 WRBUF w = wrbuf_alloc();
629 wrbuf_printf(w, "Present response multipleNSD %s: ",
630 cl->database->database->url);
632 diag_to_wrbuf(recs->u.multipleNonSurDiagnostics->diagRecs,
633 recs->u.multipleNonSurDiagnostics->num_diagRecs,
635 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
636 cl->state = Client_Error;
639 else if (recs && !*r->presentStatus && cl->state != Client_Error)
641 yaz_log(YLOG_DEBUG, "Good Present response %s",
642 cl->database->database->url);
644 // we can mix show raw and normal show ..
645 if (cl->show_raw && cl->show_raw->active)
647 cl->show_raw->active = 0; // no longer active
648 ingest_raw_records(cl, recs);
651 ingest_records(cl, recs);
652 cl->state = Client_Idle;
654 else if (*r->presentStatus)
656 yaz_log(YLOG_WARN, "Bad Present response %s",
657 cl->database->database->url);
658 cl->state = Client_Error;
659 client_show_raw_error(cl, "bad present response");
663 void client_close_response(struct client *cl, Z_APDU *a)
665 struct connection *co = cl->connection;
666 /* Z_Close *r = a->u.close; */
668 yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
670 cl->state = Client_Failed;
671 connection_destroy(co);
674 int client_is_our_response(struct client *cl)
676 struct session *se = client_get_session(cl);
678 if (cl && (cl->requestid == se->requestid ||
679 cl->state == Client_Initializing))
684 // Set authentication token in init if one is set for the client
685 // TODO: Extend this to handle other schemes than open (should be simple)
686 static void init_authentication(struct client *cl, Z_InitRequest *req)
688 struct session_database *sdb = client_get_database(cl);
689 char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
693 struct connection *co = client_get_connection(cl);
694 struct session *se = client_get_session(cl);
695 Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
697 idAuth->which = Z_IdAuthentication_open;
698 idAuth->u.open = auth;
699 req->idAuthentication = idAuth;
700 connection_set_authentication(co, nmem_strdup(se->session_nmem, auth));
704 static void init_zproxy(struct client *cl, Z_InitRequest *req)
706 struct session_database *sdb = client_get_database(cl);
707 char *ztarget = sdb->database->url;
708 //char *ztarget = sdb->url;
709 char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY);
712 yaz_oi_set_string_oid(&req->otherInfo,
713 global_parameters.odr_out,
714 yaz_oid_userinfo_proxy,
719 static void client_init_request(struct client *cl)
721 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
723 a->u.initRequest->implementationId = global_parameters.implementationId;
724 a->u.initRequest->implementationName = global_parameters.implementationName;
725 a->u.initRequest->implementationVersion =
726 global_parameters.implementationVersion;
727 ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
728 ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
729 ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
731 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
732 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
733 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
735 init_authentication(cl, a->u.initRequest);
736 init_zproxy(cl, a->u.initRequest);
738 if (send_apdu(cl, a) >= 0)
739 client_set_state(cl, Client_Initializing);
741 client_set_state(cl, Client_Error);
742 odr_reset(global_parameters.odr_out);
745 void client_continue(struct client *cl)
747 if (cl->state == Client_Connected) {
748 client_init_request(cl);
751 if (cl->state == Client_Idle)
753 struct session *se = client_get_session(cl);
754 if (cl->requestid != se->requestid && cl->pquery) {
755 // we'll have to abort this because result set is to be deleted
756 client_show_raw_cancel(cl);
757 client_send_search(cl);
759 else if (cl->show_raw)
761 client_send_raw_present(cl);
763 else if (cl->hits > 0 && cl->records < global_parameters.toget &&
764 cl->records < cl->hits) {
765 client_send_present(cl);
770 struct client *client_create(void)
776 client_freelist = client_freelist->next;
779 r = xmalloc(sizeof(struct client));
789 r->state = Client_Disconnected;
795 void client_destroy(struct client *c)
797 struct session *se = c->session;
798 if (c == se->clients)
799 se->clients = c->next;
803 for (cc = se->clients; cc && cc->next != c; cc = cc->next)
811 connection_release(c->connection);
812 c->next = client_freelist;
816 void client_set_connection(struct client *cl, struct connection *con)
818 cl->connection = con;
821 void client_disconnect(struct client *cl)
823 if (cl->state != Client_Idle)
824 cl->state = Client_Disconnected;
825 client_set_connection(cl, 0);
828 // Extract terms from query into null-terminated termlist
829 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
833 pull_terms(nmem, query, termlist, &num);
837 // Initialize CCL map for a target
838 static CCL_bibset prepare_cclmap(struct client *cl)
840 struct session_database *sdb = client_get_database(cl);
847 for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
849 char *p = strchr(s->name + 3, ':');
852 yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
857 ccl_qual_fitem(res, s->value, p);
862 // Parse the query given the settings specific to this client
863 int client_parse_query(struct client *cl, const char *query)
865 struct session *se = client_get_session(cl);
866 struct ccl_rpn_node *cn;
868 CCL_bibset ccl_map = prepare_cclmap(cl);
872 cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
873 ccl_qual_rm(&ccl_map);
876 cl->state = Client_Error;
877 yaz_log(YLOG_WARN, "Failed to parse query for %s",
878 client_get_database(cl)->database->url);
881 wrbuf_rewind(se->wrbuf);
882 ccl_pquery(se->wrbuf, cn);
884 cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
888 // Initialize relevance structure with query terms
890 extract_terms(se->nmem, cn, p);
891 se->relevance = relevance_create(client_get_database(cl)->pct,
892 se->nmem, (const char **) p,
893 se->expected_maxrecs);
900 void client_set_session(struct client *cl, struct session *se)
903 cl->next = se->clients;
907 int client_is_active(struct client *cl)
909 if (cl->connection && (cl->state == Client_Connecting ||
910 cl->state == Client_Initializing ||
911 cl->state == Client_Searching ||
912 cl->state == Client_Presenting))
917 struct client *client_next_in_session(struct client *cl)
925 int client_get_hits(struct client *cl)
930 int client_get_num_records(struct client *cl)
935 int client_get_diagnostic(struct client *cl)
937 return cl->diagnostic;
940 void client_set_database(struct client *cl, struct session_database *db)
945 struct host *client_get_host(struct client *cl)
947 return client_get_database(cl)->database->host;
950 const char *client_get_url(struct client *cl)
952 return client_get_database(cl)->database->url;
958 * indent-tabs-mode: nil
960 * vim: shiftwidth=4 tabstop=8 expandtab