1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2010 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "filter_virt_db.hpp"
22 #include <metaproxy/package.hpp>
24 #include <boost/thread/mutex.hpp>
25 #include <boost/thread/condition.hpp>
26 #include <boost/shared_ptr.hpp>
28 #include <metaproxy/util.hpp>
31 #include <yaz/otherinfo.h>
32 #include <yaz/diagbib1.h>
33 #include <yaz/match_glob.h>
38 namespace mp = metaproxy_1;
39 namespace yf = mp::filter;
41 namespace metaproxy_1 {
44 struct VirtualDB::Set {
45 Set(BackendPtr b, std::string setname);
50 std::string m_setname;
52 struct VirtualDB::Map {
53 Map(std::string database, std::list<std::string> targets, std::string route);
55 bool match(const std::string db) const;
56 std::string m_dbpattern;
57 std::list<std::string> m_targets;
60 struct VirtualDB::Backend {
61 mp::Session m_backend_session;
62 std::list<std::string> m_frontend_databases;
63 std::list<std::string> m_targets;
65 bool m_named_result_sets;
68 struct VirtualDB::Frontend {
71 mp::Session m_session;
74 yazpp_1::GDU m_init_gdu;
75 std::list<BackendPtr> m_backend_list;
76 std::map<std::string,VirtualDB::Set> m_sets;
78 void search(Package &package, Z_APDU *apdu);
79 void present(Package &package, Z_APDU *apdu);
80 void scan(Package &package, Z_APDU *apdu);
82 void close(Package &package);
83 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
85 void fixup_package(Package &p, BackendPtr b);
86 void fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
88 void fixup_npr_records(ODR odr, Z_Records *records,
91 BackendPtr lookup_backend_from_databases(
92 std::list<std::string> databases);
93 BackendPtr create_backend_from_databases(
94 std::list<std::string> databases,
96 std::string &failing_database);
98 BackendPtr init_backend(std::list<std::string> database,
100 int &error_code, std::string &addinfo);
103 class VirtualDB::Rep {
104 friend class VirtualDB;
105 friend struct Frontend;
107 FrontendPtr get_frontend(Package &package);
108 void release_frontend(Package &package);
110 std::list<VirtualDB::Map>m_maps;
111 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
112 boost::mutex m_mutex;
113 boost::condition m_cond_session_ready;
114 std::map<mp::Session, FrontendPtr> m_clients;
120 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::lookup_backend_from_databases(
121 std::list<std::string> databases)
123 std::list<BackendPtr>::const_iterator map_it;
124 map_it = m_backend_list.begin();
125 for (; map_it != m_backend_list.end(); map_it++)
126 if ((*map_it)->m_frontend_databases == databases)
132 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::create_backend_from_databases(
133 std::list<std::string> databases, int &error_code, std::string &addinfo)
135 BackendPtr b(new Backend);
136 std::list<std::string>::const_iterator db_it = databases.begin();
138 b->m_number_of_sets = 0;
139 b->m_frontend_databases = databases;
140 b->m_named_result_sets = false;
142 bool first_route = true;
144 std::map<std::string,bool> targets_dedup;
145 for (; db_it != databases.end(); db_it++)
147 std::list<VirtualDB::Map>::const_iterator map_it;
148 map_it = m_p->m_maps.begin();
149 while (map_it != m_p->m_maps.end())
151 if (map_it->match(*db_it))
156 if (map_it == m_p->m_maps.end()) // database not found
158 error_code = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
163 std::list<std::string>::const_iterator t_it =
164 map_it->m_targets.begin();
165 for (; t_it != map_it->m_targets.end(); t_it++) {
166 if (!targets_dedup[*t_it])
168 targets_dedup[*t_it] = true;
169 b->m_targets.push_back(*t_it);
173 // see if we have a route conflict.
174 if (!first_route && b->m_route != map_it->m_route)
176 // we have a conflict..
177 error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
181 b->m_route = map_it->m_route;
187 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend(
188 std::list<std::string> databases, mp::Package &package,
189 int &error_code, std::string &addinfo)
191 BackendPtr b = create_backend_from_databases(databases, error_code,
195 Package init_package(b->m_backend_session, package.origin());
196 init_package.copy_filter(package);
200 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
202 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr,
204 Z_InitRequest *req = init_apdu->u.initRequest;
206 // copy stuff from Frontend Init Request
207 Z_GDU *org_gdu = m_init_gdu.get();
208 Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
210 req->idAuthentication = org_init->idAuthentication;
211 req->implementationId = org_init->implementationId;
212 req->implementationName = org_init->implementationName;
213 req->implementationVersion = org_init->implementationVersion;
215 ODR_MASK_SET(req->options, Z_Options_search);
216 ODR_MASK_SET(req->options, Z_Options_present);
217 ODR_MASK_SET(req->options, Z_Options_namedResultSets);
218 ODR_MASK_SET(req->options, Z_Options_scan);
220 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
221 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
222 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
224 init_package.request() = init_apdu;
226 init_package.move(b->m_route); // sending init
228 Z_GDU *gdu = init_package.response().get();
229 // we hope to get an init response
230 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
233 Z_InitResponse *res = gdu->u.z3950->u.initResponse;
234 if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
236 b->m_named_result_sets = true;
240 m_backend_list.push_back(b);
244 error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
245 mp::util::get_init_diagnostics(res, error_code, addinfo);
247 if (!init_package.session().is_closed())
249 Package close_package(b->m_backend_session, package.origin());
250 close_package.copy_filter(package);
251 close_package.session().close();
252 close_package.move(b->m_route); // closing it
258 void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
260 Z_SearchRequest *req = apdu_req->u.searchRequest;
262 std::string resultSetId = req->resultSetName;
265 std::list<std::string> databases;
267 for (i = 0; i<req->num_databaseNames; i++)
268 databases.push_back(req->databaseNames[i]);
270 BackendPtr b; // null for now
271 Sets_it sets_it = m_sets.find(req->resultSetName);
272 if (sets_it != m_sets.end())
274 // result set already exist
275 // if replace indicator is off: we return diagnostic if
276 // result set already exist.
277 if (*req->replaceIndicator == 0)
280 odr.create_searchResponse(
282 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
284 package.response() = apdu;
288 sets_it->second.m_backend->m_number_of_sets--;
290 // pick up any existing backend with a database match
291 std::list<BackendPtr>::const_iterator map_it;
292 map_it = m_backend_list.begin();
293 for (; map_it != m_backend_list.end(); map_it++)
295 BackendPtr tmp = *map_it;
296 if (tmp->m_frontend_databases == databases)
299 if (map_it != m_backend_list.end())
306 // pick up any existing database with named result sets ..
307 // or one which has no result sets.. yet.
308 std::list<BackendPtr>::const_iterator map_it;
309 map_it = m_backend_list.begin();
310 for (; map_it != m_backend_list.end(); map_it++)
312 BackendPtr tmp = *map_it;
313 if (tmp->m_frontend_databases == databases &&
314 (tmp->m_named_result_sets ||
315 tmp->m_number_of_sets == 0))
318 if (map_it != m_backend_list.end())
321 if (!b) // no backend yet. Must create a new one
325 b = init_backend(databases, package, error_code, addinfo);
328 // did not get a backend (unavailable somehow?)
331 odr.create_searchResponse(
332 apdu_req, error_code, addinfo.c_str());
333 package.response() = apdu;
337 m_sets.erase(req->resultSetName);
338 // sending search to backend
339 Package search_package(b->m_backend_session, package.origin());
341 search_package.copy_filter(package);
343 std::string backend_setname;
344 if (b->m_named_result_sets)
346 backend_setname = std::string(req->resultSetName);
350 backend_setname = "default";
351 req->resultSetName = odr_strdup(odr, backend_setname.c_str());
354 // pick first targets spec and move the databases from it ..
355 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
356 if (t_it != b->m_targets.end())
358 mp::util::set_databases_from_zurl(odr, *t_it,
359 &req->num_databaseNames,
360 &req->databaseNames);
363 *req->replaceIndicator = 1;
365 search_package.request() = yazpp_1::GDU(apdu_req);
367 search_package.move(b->m_route);
369 if (search_package.session().is_closed())
371 package.response() = search_package.response();
372 package.session().close();
375 b->m_number_of_sets++;
377 m_sets[resultSetId] = VirtualDB::Set(b, backend_setname);
378 fixup_package(search_package, b);
379 package.response() = search_package.response();
382 yf::VirtualDB::Frontend::Frontend(Rep *rep)
385 m_is_virtual = false;
388 void yf::VirtualDB::Frontend::close(mp::Package &package)
390 std::list<BackendPtr>::const_iterator b_it;
392 for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
394 (*b_it)->m_backend_session.close();
395 Package close_package((*b_it)->m_backend_session, package.origin());
396 close_package.copy_filter(package);
397 close_package.move((*b_it)->m_route);
399 m_backend_list.clear();
402 yf::VirtualDB::Frontend::~Frontend()
406 yf::VirtualDB::FrontendPtr yf::VirtualDB::Rep::get_frontend(mp::Package &package)
408 boost::mutex::scoped_lock lock(m_mutex);
410 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
414 it = m_clients.find(package.session());
415 if (it == m_clients.end())
418 if (!it->second->m_in_use)
420 it->second->m_in_use = true;
423 m_cond_session_ready.wait(lock);
425 FrontendPtr f(new Frontend(this));
426 m_clients[package.session()] = f;
431 void yf::VirtualDB::Rep::release_frontend(mp::Package &package)
433 boost::mutex::scoped_lock lock(m_mutex);
434 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
436 it = m_clients.find(package.session());
437 if (it != m_clients.end())
439 if (package.session().is_closed())
441 it->second->close(package);
446 it->second->m_in_use = false;
448 m_cond_session_ready.notify_all();
452 yf::VirtualDB::Set::Set(BackendPtr b, std::string setname)
453 : m_backend(b), m_setname(setname)
458 yf::VirtualDB::Set::Set()
463 yf::VirtualDB::Set::~Set()
467 yf::VirtualDB::Map::Map(std::string database,
468 std::list<std::string> targets, std::string route)
469 : m_dbpattern(database), m_targets(targets), m_route(route)
473 yf::VirtualDB::Map::Map()
477 bool yf::VirtualDB::Map::match(const std::string db) const
479 std::string norm_db = mp::util::database_name_normalize(db);
480 if (yaz_match_glob(m_dbpattern.c_str(), norm_db.c_str()))
485 yf::VirtualDB::VirtualDB() : m_p(new VirtualDB::Rep)
487 m_p->pass_vhosts = false;
490 yf::VirtualDB::~VirtualDB() {
493 void yf::VirtualDB::Frontend::fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
496 if (npr->databaseName)
498 std::string b_database = std::string(npr->databaseName);
500 // consider each of the frontend databases..
501 std::list<std::string>::const_iterator db_it;
502 for (db_it = b->m_frontend_databases.begin();
503 db_it != b->m_frontend_databases.end(); db_it++)
505 // see which target it corresponds to.. (if any)
506 std::list<VirtualDB::Map>::const_iterator map_it =
508 while (map_it != m_p->m_maps.end())
510 if (map_it->match(*db_it))
514 if (map_it != m_p->m_maps.end())
516 std::list<std::string>::const_iterator t
517 = map_it->m_targets.begin();
518 while (t != map_it->m_targets.end())
520 if (*t == b_database)
522 npr->databaseName = odr_strdup(odr, (*db_it).c_str());
530 db_it = b->m_frontend_databases.begin();
531 if (db_it != b->m_frontend_databases.end())
533 std::string database = *db_it;
534 npr->databaseName = odr_strdup(odr, database.c_str());
539 void yf::VirtualDB::Frontend::fixup_npr_records(ODR odr, Z_Records *records,
542 if (records && records->which == Z_Records_DBOSD)
544 Z_NamePlusRecordList *nprlist = records->u.databaseOrSurDiagnostics;
546 for (i = 0; i < nprlist->num_records; i++)
548 fixup_npr_record(odr, nprlist->records[i], b);
553 void yf::VirtualDB::Frontend::fixup_package(mp::Package &p, BackendPtr b)
555 Z_GDU *gdu = p.response().get();
558 if (gdu && gdu->which == Z_GDU_Z3950)
560 Z_APDU *apdu = gdu->u.z3950;
561 if (apdu->which == Z_APDU_presentResponse)
563 fixup_npr_records(odr, apdu->u.presentResponse->records, b);
566 else if (apdu->which == Z_APDU_searchResponse)
568 fixup_npr_records(odr, apdu->u.searchResponse->records, b);
574 void yf::VirtualDB::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
576 Z_PresentRequest *req = apdu_req->u.presentRequest;
577 std::string resultSetId = req->resultSetId;
580 Sets_it sets_it = m_sets.find(resultSetId);
581 if (sets_it == m_sets.end())
584 odr.create_presentResponse(
586 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
587 resultSetId.c_str());
588 package.response() = apdu;
592 new mp::Session(sets_it->second.m_backend->m_backend_session);
594 // sending present to backend
595 Package present_package(*id, package.origin());
596 present_package.copy_filter(package);
598 req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
600 present_package.request() = yazpp_1::GDU(apdu_req);
602 present_package.move(sets_it->second.m_backend->m_route);
604 fixup_package(present_package, sets_it->second.m_backend);
606 if (present_package.session().is_closed())
608 package.response() = present_package.response();
609 package.session().close();
614 package.response() = present_package.response();
619 void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
621 Z_ScanRequest *req = apdu_req->u.scanRequest;
625 std::list<std::string> databases;
627 for (i = 0; i<req->num_databaseNames; i++)
628 databases.push_back(req->databaseNames[i]);
631 // pick up any existing backend with a database match
632 std::list<BackendPtr>::const_iterator map_it;
633 map_it = m_backend_list.begin();
634 for (; map_it != m_backend_list.end(); map_it++)
636 BackendPtr tmp = *map_it;
637 if (tmp->m_frontend_databases == databases)
640 if (map_it != m_backend_list.end())
642 if (!b) // no backend yet. Must create a new one
646 b = init_backend(databases, package, error_code, addinfo);
649 // did not get a backend (unavailable somehow?)
651 odr.create_scanResponse(
652 apdu_req, error_code, addinfo.c_str());
653 package.response() = apdu;
658 // sending scan to backend
659 Package scan_package(b->m_backend_session, package.origin());
661 scan_package.copy_filter(package);
663 // pick first targets spec and move the databases from it ..
664 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
665 if (t_it != b->m_targets.end())
667 mp::util::set_databases_from_zurl(odr, *t_it,
668 &req->num_databaseNames,
669 &req->databaseNames);
671 scan_package.request() = yazpp_1::GDU(apdu_req);
673 scan_package.move(b->m_route);
675 if (scan_package.session().is_closed())
677 package.response() = scan_package.response();
678 package.session().close();
681 package.response() = scan_package.response();
685 void yf::VirtualDB::add_map_db2targets(std::string db,
686 std::list<std::string> targets,
689 m_p->m_maps.push_back(
690 VirtualDB::Map(mp::util::database_name_normalize(db), targets, route));
694 void yf::VirtualDB::add_map_db2target(std::string db,
698 std::list<std::string> targets;
699 targets.push_back(target);
701 add_map_db2targets(db, targets, route);
704 void yf::VirtualDB::process(mp::Package &package) const
706 FrontendPtr f = m_p->get_frontend(package);
708 Z_GDU *gdu = package.request().get();
710 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
711 Z_APDU_initRequest && !f->m_is_virtual)
713 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
715 std::list<std::string> vhosts;
716 mp::util::get_vhost_otherinfo(req->otherInfo, vhosts);
718 if (vhosts.size() > 0 && m_p->pass_vhosts)
727 Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
728 Z_InitResponse *resp = apdu->u.initResponse;
731 static const int masks[] = {
734 Z_Options_namedResultSets,
738 for (i = 0; masks[i] != -1; i++)
739 if (ODR_MASK_GET(req->options, masks[i]))
740 ODR_MASK_SET(resp->options, masks[i]);
742 static const int versions[] = {
748 for (i = 0; versions[i] != -1; i++)
749 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
750 ODR_MASK_SET(resp->protocolVersion, versions[i]);
754 package.response() = apdu;
755 f->m_is_virtual = true;
758 else if (!f->m_is_virtual)
760 else if (gdu && gdu->which == Z_GDU_Z3950)
762 Z_APDU *apdu = gdu->u.z3950;
763 if (apdu->which == Z_APDU_initRequest)
767 package.response() = odr.create_close(
769 Z_Close_protocolError,
772 package.session().close();
774 else if (apdu->which == Z_APDU_searchRequest)
776 f->search(package, apdu);
778 else if (apdu->which == Z_APDU_presentRequest)
780 f->present(package, apdu);
782 else if (apdu->which == Z_APDU_scanRequest)
784 f->scan(package, apdu);
786 else if (apdu->which == Z_APDU_close)
788 package.session().close();
794 package.response() = odr.create_close(
795 apdu, Z_Close_protocolError,
796 "unsupported APDU in filter_virt_db");
798 package.session().close();
801 m_p->release_frontend(package);
805 void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only)
807 for (ptr = ptr->children; ptr; ptr = ptr->next)
809 if (ptr->type != XML_ELEMENT_NODE)
811 if (!strcmp((const char *) ptr->name, "pass-vhosts"))
813 m_p->pass_vhosts = mp::xml::get_bool(ptr->children, false);
815 else if (!strcmp((const char *) ptr->name, "virtual"))
817 std::string database;
818 std::list<std::string> targets;
819 xmlNode *v_node = ptr->children;
820 for (; v_node; v_node = v_node->next)
822 if (v_node->type != XML_ELEMENT_NODE)
825 if (mp::xml::is_element_mp(v_node, "database"))
826 database = mp::xml::get_text(v_node);
827 else if (mp::xml::is_element_mp(v_node, "target"))
828 targets.push_back(mp::xml::get_text(v_node));
830 throw mp::filter::FilterException
832 + std::string((const char *) v_node->name)
833 + " in virtual section"
836 std::string route = mp::xml::get_route(ptr);
837 add_map_db2targets(database, targets, route);
841 throw mp::filter::FilterException
843 + std::string((const char *) ptr->name)
844 + " in virt_db filter");
849 static mp::filter::Base* filter_creator()
851 return new mp::filter::VirtualDB;
855 struct metaproxy_1_filter_struct metaproxy_1_filter_virt_db = {
866 * c-file-style: "Stroustrup"
867 * indent-tabs-mode: nil
869 * vim: shiftwidth=4 tabstop=8 expandtab