1 /* $Id: filter_virt_db.cpp,v 1.52 2007-08-13 10:19:16 adam Exp $
2 Copyright (c) 2005-2007, Index Data.
4 This file is part of Metaproxy.
6 Metaproxy is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Metaproxy; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
21 /* $Id: filter_virt_db.cpp,v 1.52 2007-08-13 10:19:16 adam Exp $
22 Copyright (c) 2005-2007, Index Data.
24 See the LICENSE file for details
30 #include "package.hpp"
32 #include <boost/thread/mutex.hpp>
33 #include <boost/thread/condition.hpp>
34 #include <boost/shared_ptr.hpp>
37 #include "filter_virt_db.hpp"
40 #include <yaz/otherinfo.h>
41 #include <yaz/diagbib1.h>
46 namespace mp = metaproxy_1;
47 namespace yf = mp::filter;
49 namespace metaproxy_1 {
52 struct VirtualDB::Set {
53 Set(BackendPtr b, std::string setname);
58 std::string m_setname;
60 struct VirtualDB::Map {
61 Map(std::list<std::string> targets, std::string route);
63 std::list<std::string> m_targets;
66 struct VirtualDB::Backend {
67 mp::Session m_backend_session;
68 std::list<std::string> m_frontend_databases;
69 std::list<std::string> m_targets;
71 bool m_named_result_sets;
74 struct VirtualDB::Frontend {
77 mp::Session m_session;
80 yazpp_1::GDU m_init_gdu;
81 std::list<BackendPtr> m_backend_list;
82 std::map<std::string,VirtualDB::Set> m_sets;
84 void search(Package &package, Z_APDU *apdu);
85 void present(Package &package, Z_APDU *apdu);
86 void scan(Package &package, Z_APDU *apdu);
88 void close(Package &package);
89 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
91 void fixup_package(Package &p, BackendPtr b);
92 void fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
94 void fixup_npr_records(ODR odr, Z_Records *records,
97 BackendPtr lookup_backend_from_databases(
98 std::list<std::string> databases);
99 BackendPtr create_backend_from_databases(
100 std::list<std::string> databases,
102 std::string &failing_database);
104 BackendPtr init_backend(std::list<std::string> database,
106 int &error_code, std::string &addinfo);
109 class VirtualDB::Rep {
110 friend class VirtualDB;
111 friend struct Frontend;
113 FrontendPtr get_frontend(Package &package);
114 void release_frontend(Package &package);
116 std::map<std::string, VirtualDB::Map>m_maps;
117 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
118 boost::mutex m_mutex;
119 boost::condition m_cond_session_ready;
120 std::map<mp::Session, FrontendPtr> m_clients;
125 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::lookup_backend_from_databases(
126 std::list<std::string> databases)
128 std::list<BackendPtr>::const_iterator map_it;
129 map_it = m_backend_list.begin();
130 for (; map_it != m_backend_list.end(); map_it++)
131 if ((*map_it)->m_frontend_databases == databases)
137 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::create_backend_from_databases(
138 std::list<std::string> databases, int &error_code, std::string &addinfo)
140 BackendPtr b(new Backend);
141 std::list<std::string>::const_iterator db_it = databases.begin();
143 b->m_number_of_sets = 0;
144 b->m_frontend_databases = databases;
145 b->m_named_result_sets = false;
147 bool first_route = true;
149 std::map<std::string,bool> targets_dedup;
150 for (; db_it != databases.end(); db_it++)
152 std::map<std::string, VirtualDB::Map>::iterator map_it;
153 map_it = m_p->m_maps.find(mp::util::database_name_normalize(*db_it));
154 if (map_it == m_p->m_maps.end()) // database not found
156 error_code = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
161 std::list<std::string>::const_iterator t_it =
162 map_it->second.m_targets.begin();
163 for (; t_it != map_it->second.m_targets.end(); t_it++)
164 targets_dedup[*t_it] = true;
166 // see if we have a route conflict.
167 if (!first_route && b->m_route != map_it->second.m_route)
169 // we have a conflict..
170 error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
174 b->m_route = map_it->second.m_route;
177 std::map<std::string,bool>::const_iterator tm_it = targets_dedup.begin();
178 for (; tm_it != targets_dedup.end(); tm_it++)
179 b->m_targets.push_back(tm_it->first);
184 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend(
185 std::list<std::string> databases, mp::Package &package,
186 int &error_code, std::string &addinfo)
188 BackendPtr b = create_backend_from_databases(databases, error_code,
192 Package init_package(b->m_backend_session, package.origin());
193 init_package.copy_filter(package);
197 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
199 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr,
201 Z_InitRequest *req = init_apdu->u.initRequest;
203 // copy stuff from Frontend Init Request
204 Z_GDU *org_gdu = m_init_gdu.get();
205 Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
207 req->idAuthentication = org_init->idAuthentication;
208 req->implementationId = org_init->implementationId;
209 req->implementationName = org_init->implementationName;
210 req->implementationVersion = org_init->implementationVersion;
212 ODR_MASK_SET(req->options, Z_Options_search);
213 ODR_MASK_SET(req->options, Z_Options_present);
214 ODR_MASK_SET(req->options, Z_Options_namedResultSets);
215 ODR_MASK_SET(req->options, Z_Options_scan);
217 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
218 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
219 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
221 init_package.request() = init_apdu;
223 init_package.move(b->m_route); // sending init
225 Z_GDU *gdu = init_package.response().get();
226 // we hope to get an init response
227 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
230 Z_InitResponse *res = gdu->u.z3950->u.initResponse;
231 if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
233 b->m_named_result_sets = true;
237 error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
238 mp::util::get_init_diagnostics(res, error_code, addinfo);
245 error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
246 // addinfo = database;
250 if (init_package.session().is_closed())
252 error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
253 // addinfo = database;
258 m_backend_list.push_back(b);
262 void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
264 Z_SearchRequest *req = apdu_req->u.searchRequest;
266 std::string resultSetId = req->resultSetName;
269 std::list<std::string> databases;
271 for (i = 0; i<req->num_databaseNames; i++)
272 databases.push_back(req->databaseNames[i]);
274 BackendPtr b; // null for now
275 Sets_it sets_it = m_sets.find(req->resultSetName);
276 if (sets_it != m_sets.end())
278 // result set already exist
279 // if replace indicator is off: we return diagnostic if
280 // result set already exist.
281 if (*req->replaceIndicator == 0)
284 odr.create_searchResponse(
286 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
288 package.response() = apdu;
292 sets_it->second.m_backend->m_number_of_sets--;
294 // pick up any existing backend with a database match
295 std::list<BackendPtr>::const_iterator map_it;
296 map_it = m_backend_list.begin();
297 for (; map_it != m_backend_list.end(); map_it++)
299 BackendPtr tmp = *map_it;
300 if (tmp->m_frontend_databases == databases)
303 if (map_it != m_backend_list.end())
310 // pick up any existing database with named result sets ..
311 // or one which has no result sets.. yet.
312 std::list<BackendPtr>::const_iterator map_it;
313 map_it = m_backend_list.begin();
314 for (; map_it != m_backend_list.end(); map_it++)
316 BackendPtr tmp = *map_it;
317 if (tmp->m_frontend_databases == databases &&
318 (tmp->m_named_result_sets ||
319 tmp->m_number_of_sets == 0))
322 if (map_it != m_backend_list.end())
325 if (!b) // no backend yet. Must create a new one
329 b = init_backend(databases, package, error_code, addinfo);
332 // did not get a backend (unavailable somehow?)
335 odr.create_searchResponse(
336 apdu_req, error_code, addinfo.c_str());
337 package.response() = apdu;
341 m_sets.erase(req->resultSetName);
342 // sending search to backend
343 Package search_package(b->m_backend_session, package.origin());
345 search_package.copy_filter(package);
347 std::string backend_setname;
348 if (b->m_named_result_sets)
350 backend_setname = std::string(req->resultSetName);
354 backend_setname = "default";
355 req->resultSetName = odr_strdup(odr, backend_setname.c_str());
358 // pick first targets spec and move the databases from it ..
359 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
360 if (t_it != b->m_targets.end())
362 mp::util::set_databases_from_zurl(odr, *t_it,
363 &req->num_databaseNames,
364 &req->databaseNames);
367 *req->replaceIndicator = 1;
369 search_package.request() = yazpp_1::GDU(apdu_req);
371 search_package.move(b->m_route);
373 if (search_package.session().is_closed())
375 package.response() = search_package.response();
376 package.session().close();
379 b->m_number_of_sets++;
381 m_sets[resultSetId] = VirtualDB::Set(b, backend_setname);
382 fixup_package(search_package, b);
383 package.response() = search_package.response();
386 yf::VirtualDB::Frontend::Frontend(Rep *rep)
389 m_is_virtual = false;
392 void yf::VirtualDB::Frontend::close(mp::Package &package)
394 std::list<BackendPtr>::const_iterator b_it;
396 for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
398 (*b_it)->m_backend_session.close();
399 Package close_package((*b_it)->m_backend_session, package.origin());
400 close_package.copy_filter(package);
401 close_package.move((*b_it)->m_route);
403 m_backend_list.clear();
406 yf::VirtualDB::Frontend::~Frontend()
410 yf::VirtualDB::FrontendPtr yf::VirtualDB::Rep::get_frontend(mp::Package &package)
412 boost::mutex::scoped_lock lock(m_mutex);
414 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
418 it = m_clients.find(package.session());
419 if (it == m_clients.end())
422 if (!it->second->m_in_use)
424 it->second->m_in_use = true;
427 m_cond_session_ready.wait(lock);
429 FrontendPtr f(new Frontend(this));
430 m_clients[package.session()] = f;
435 void yf::VirtualDB::Rep::release_frontend(mp::Package &package)
437 boost::mutex::scoped_lock lock(m_mutex);
438 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
440 it = m_clients.find(package.session());
441 if (it != m_clients.end())
443 if (package.session().is_closed())
445 it->second->close(package);
450 it->second->m_in_use = false;
452 m_cond_session_ready.notify_all();
456 yf::VirtualDB::Set::Set(BackendPtr b, std::string setname)
457 : m_backend(b), m_setname(setname)
462 yf::VirtualDB::Set::Set()
467 yf::VirtualDB::Set::~Set()
471 yf::VirtualDB::Map::Map(std::list<std::string> targets, std::string route)
472 : m_targets(targets), m_route(route)
476 yf::VirtualDB::Map::Map()
480 yf::VirtualDB::VirtualDB() : m_p(new VirtualDB::Rep)
484 yf::VirtualDB::~VirtualDB() {
487 void yf::VirtualDB::Frontend::fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
490 if (npr->databaseName)
492 std::string b_database = std::string(npr->databaseName);
494 // consider each of the frontend databases..
495 std::list<std::string>::const_iterator db_it;
496 for (db_it = b->m_frontend_databases.begin();
497 db_it != b->m_frontend_databases.end(); db_it++)
499 // see which target it corresponds to.. (if any)
500 std::map<std::string,VirtualDB::Map>::const_iterator map_it;
502 map_it = m_p->m_maps.find(mp::util::database_name_normalize(*db_it));
503 if (map_it != m_p->m_maps.end())
505 VirtualDB::Map m = map_it->second;
507 std::list<std::string>::const_iterator t;
508 for (t = m.m_targets.begin(); t != m.m_targets.end(); t++)
510 if (*t == b_database)
512 npr->databaseName = odr_strdup(odr, (*db_it).c_str());
519 db_it = b->m_frontend_databases.begin();
520 if (db_it != b->m_frontend_databases.end())
522 std::string database = *db_it;
523 npr->databaseName = odr_strdup(odr, database.c_str());
528 void yf::VirtualDB::Frontend::fixup_npr_records(ODR odr, Z_Records *records,
531 if (records && records->which == Z_Records_DBOSD)
533 Z_NamePlusRecordList *nprlist = records->u.databaseOrSurDiagnostics;
535 for (i = 0; i < nprlist->num_records; i++)
537 fixup_npr_record(odr, nprlist->records[i], b);
542 void yf::VirtualDB::Frontend::fixup_package(mp::Package &p, BackendPtr b)
544 Z_GDU *gdu = p.response().get();
547 if (gdu && gdu->which == Z_GDU_Z3950)
549 Z_APDU *apdu = gdu->u.z3950;
550 if (apdu->which == Z_APDU_presentResponse)
552 fixup_npr_records(odr, apdu->u.presentResponse->records, b);
555 else if (apdu->which == Z_APDU_searchResponse)
557 fixup_npr_records(odr, apdu->u.searchResponse->records, b);
563 void yf::VirtualDB::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
565 Z_PresentRequest *req = apdu_req->u.presentRequest;
566 std::string resultSetId = req->resultSetId;
569 Sets_it sets_it = m_sets.find(resultSetId);
570 if (sets_it == m_sets.end())
573 odr.create_presentResponse(
575 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
576 resultSetId.c_str());
577 package.response() = apdu;
581 new mp::Session(sets_it->second.m_backend->m_backend_session);
583 // sending present to backend
584 Package present_package(*id, package.origin());
585 present_package.copy_filter(package);
587 req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
589 present_package.request() = yazpp_1::GDU(apdu_req);
591 present_package.move(sets_it->second.m_backend->m_route);
593 fixup_package(present_package, sets_it->second.m_backend);
595 if (present_package.session().is_closed())
597 package.response() = present_package.response();
598 package.session().close();
603 package.response() = present_package.response();
608 void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
610 Z_ScanRequest *req = apdu_req->u.scanRequest;
614 std::list<std::string> databases;
616 for (i = 0; i<req->num_databaseNames; i++)
617 databases.push_back(req->databaseNames[i]);
620 // pick up any existing backend with a database match
621 std::list<BackendPtr>::const_iterator map_it;
622 map_it = m_backend_list.begin();
623 for (; map_it != m_backend_list.end(); map_it++)
625 BackendPtr tmp = *map_it;
626 if (tmp->m_frontend_databases == databases)
629 if (map_it != m_backend_list.end())
631 if (!b) // no backend yet. Must create a new one
635 b = init_backend(databases, package, error_code, addinfo);
638 // did not get a backend (unavailable somehow?)
640 odr.create_scanResponse(
641 apdu_req, error_code, addinfo.c_str());
642 package.response() = apdu;
647 // sending scan to backend
648 Package scan_package(b->m_backend_session, package.origin());
650 scan_package.copy_filter(package);
652 // pick first targets spec and move the databases from it ..
653 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
654 if (t_it != b->m_targets.end())
656 mp::util::set_databases_from_zurl(odr, *t_it,
657 &req->num_databaseNames,
658 &req->databaseNames);
660 scan_package.request() = yazpp_1::GDU(apdu_req);
662 scan_package.move(b->m_route);
664 if (scan_package.session().is_closed())
666 package.response() = scan_package.response();
667 package.session().close();
670 package.response() = scan_package.response();
674 void yf::VirtualDB::add_map_db2targets(std::string db,
675 std::list<std::string> targets,
678 m_p->m_maps[mp::util::database_name_normalize(db)]
679 = VirtualDB::Map(targets, route);
683 void yf::VirtualDB::add_map_db2target(std::string db,
687 std::list<std::string> targets;
688 targets.push_back(target);
690 m_p->m_maps[mp::util::database_name_normalize(db)]
691 = VirtualDB::Map(targets, route);
694 void yf::VirtualDB::process(mp::Package &package) const
696 FrontendPtr f = m_p->get_frontend(package);
698 Z_GDU *gdu = package.request().get();
700 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
701 Z_APDU_initRequest && !f->m_is_virtual)
703 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
705 std::list<std::string> vhosts;
706 mp::util::get_vhost_otherinfo(req->otherInfo, vhosts);
707 if (vhosts.size() == 0)
712 Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
713 Z_InitResponse *resp = apdu->u.initResponse;
716 static const int masks[] = {
719 Z_Options_namedResultSets,
723 for (i = 0; masks[i] != -1; i++)
724 if (ODR_MASK_GET(req->options, masks[i]))
725 ODR_MASK_SET(resp->options, masks[i]);
727 static const int versions[] = {
733 for (i = 0; versions[i] != -1; i++)
734 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
735 ODR_MASK_SET(resp->protocolVersion, versions[i]);
739 package.response() = apdu;
740 f->m_is_virtual = true;
745 else if (!f->m_is_virtual)
747 else if (gdu && gdu->which == Z_GDU_Z3950)
749 Z_APDU *apdu = gdu->u.z3950;
750 if (apdu->which == Z_APDU_initRequest)
754 package.response() = odr.create_close(
756 Z_Close_protocolError,
759 package.session().close();
761 else if (apdu->which == Z_APDU_searchRequest)
763 f->search(package, apdu);
765 else if (apdu->which == Z_APDU_presentRequest)
767 f->present(package, apdu);
769 else if (apdu->which == Z_APDU_scanRequest)
771 f->scan(package, apdu);
773 else if (apdu->which == Z_APDU_close)
775 package.session().close();
781 package.response() = odr.create_close(
782 apdu, Z_Close_protocolError,
783 "unsupported APDU in filter_virt_db");
785 package.session().close();
788 m_p->release_frontend(package);
792 void mp::filter::VirtualDB::configure(const xmlNode * ptr)
794 for (ptr = ptr->children; ptr; ptr = ptr->next)
796 if (ptr->type != XML_ELEMENT_NODE)
798 if (!strcmp((const char *) ptr->name, "virtual"))
800 std::string database;
801 std::list<std::string> targets;
802 xmlNode *v_node = ptr->children;
803 for (; v_node; v_node = v_node->next)
805 if (v_node->type != XML_ELEMENT_NODE)
808 if (mp::xml::is_element_mp(v_node, "database"))
809 database = mp::xml::get_text(v_node);
810 else if (mp::xml::is_element_mp(v_node, "target"))
811 targets.push_back(mp::xml::get_text(v_node));
813 throw mp::filter::FilterException
815 + std::string((const char *) v_node->name)
816 + " in virtual section"
819 std::string route = mp::xml::get_route(ptr);
820 add_map_db2targets(database, targets, route);
824 throw mp::filter::FilterException
826 + std::string((const char *) ptr->name)
827 + " in virt_db filter");
832 static mp::filter::Base* filter_creator()
834 return new mp::filter::VirtualDB;
838 struct metaproxy_1_filter_struct metaproxy_1_filter_virt_db = {
849 * indent-tabs-mode: nil
850 * c-file-style: "stroustrup"
852 * vim: shiftwidth=4 tabstop=8 expandtab