/* This file is part of Metaproxy.
- Copyright (C) 2005-2011 Index Data
+ Copyright (C) 2005-2012 Index Data
Metaproxy is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
BackendPtr m_backend;
int m_pos; // position for backend (1=first, 2=second,..
int m_start; // present request start
- PresentJob(BackendPtr ptr, int pos) :
+ PresentJob(BackendPtr ptr, int pos) :
m_backend(ptr), m_pos(pos), m_start(0) {};
};
FrontendSet(std::string setname);
std::string m_backend_database;
std::string m_vhost;
std::string m_route;
+ std::string m_auth;
void operator() (void); // thread operation
};
struct Multi::Frontend {
void present(Package &package, Z_APDU *apdu);
void scan1(Package &package, Z_APDU *apdu);
void scan2(Package &package, Z_APDU *apdu);
+ void relay_apdu(Package &package, Z_APDU *apdu);
Rep *m_p;
- };
+ };
class Multi::Map {
std::string m_target_pattern;
std::string m_route;
+ std::string m_auth;
public:
- Map(std::string pattern, std::string route) :
- m_target_pattern(pattern), m_route(route) {};
- bool match(const std::string target, std::string *ret) const {
+ Map(std::string pattern, std::string route, std::string auth) :
+ m_target_pattern(pattern), m_route(route), m_auth(auth) {};
+ bool match(const std::string target, std::string *ret,
+ std::string *auth) const {
if (yaz_match_glob(m_target_pattern.c_str(), target.c_str()))
{
*ret = m_route;
+ *auth = m_auth;
return true;
}
return false;
class Multi::Rep {
friend class Multi;
friend struct Frontend;
-
+
Rep();
FrontendPtr get_frontend(Package &package);
void release_frontend(Package &package);
boost::condition m_cond_session_ready;
std::map<mp::Session, FrontendPtr> m_clients;
bool m_hide_unavailable;
+ bool m_hide_errors;
multi_merge_type m_merge_type;
};
}
yf::Multi::Rep::Rep()
{
m_hide_unavailable = false;
+ m_hide_errors = false;
m_merge_type = round_robin;
}
boost::mutex::scoped_lock lock(m_mutex);
std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
-
+
while(true)
{
it = m_clients.find(package.session());
if (it == m_clients.end())
break;
-
+
if (!it->second->m_in_use)
{
it->second->m_in_use = true;
{
boost::mutex::scoped_lock lock(m_mutex);
std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
-
+
it = m_clients.find(package.session());
if (it != m_clients.end())
{
}
-void yf::Multi::Backend::operator() (void)
+void yf::Multi::Backend::operator() (void)
{
m_package->move(m_route);
}
std::list<BackendSet>::const_iterator bsit;
int voffset = 0;
int offset = start + i - 1;
- for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
+ for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
bsit++)
{
if (offset >= voffset && offset < voffset + bsit->m_count)
// Yes. skip until start.. Rounding off is deliberate!
min = (start-p) / no_left;
p += no_left * min;
-
+
// update positions in each set..
std::list<int>::iterator psit = pos.begin();
for (psit = pos.begin(); psit != pos.end(); psit++)
}
// skip on each set.. before "present range"..
p = p + skip;
-
+
std::list<int>::iterator psit = pos.begin();
for (psit = pos.begin(); psit != pos.end(); psit++)
*psit += min;
-
+
omin = min; // update so we consider next class (with higher count)
}
#endif
std::list<Multi::Map>::const_iterator it =
m_p->m_route_patterns.begin();
while (it != m_p->m_route_patterns.end()) {
- if (it->match(*t_it, &b->m_route))
+ if (it->match(*t_it, &b->m_route, &b->m_auth))
break;
it++;
}
}
m_is_multi = true;
- // create init request
+ // create init request
std::list<BackendPtr>::iterator bit;
for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
{
mp::odr odr;
BackendPtr b = *bit;
Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
-
+
std::list<std::string>vhost_one;
vhost_one.push_back(b->m_vhost);
mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
odr, vhost_one);
- init_apdu->u.initRequest->idAuthentication = req->idAuthentication;
-
- Z_InitRequest *req = init_apdu->u.initRequest;
-
- ODR_MASK_SET(req->options, Z_Options_search);
- ODR_MASK_SET(req->options, Z_Options_present);
- ODR_MASK_SET(req->options, Z_Options_namedResultSets);
- ODR_MASK_SET(req->options, Z_Options_scan);
-
- ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
- ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
- ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
-
+
+ Z_InitRequest *breq = init_apdu->u.initRequest;
+
+ if (b->m_auth.length())
+ {
+ breq->idAuthentication =
+ (Z_IdAuthentication *)
+ odr_malloc(odr, sizeof(*breq->idAuthentication));
+ breq->idAuthentication->which = Z_IdAuthentication_open;
+ breq->idAuthentication->u.open = odr_strdup(odr, b->m_auth.c_str());
+ }
+ else
+ breq->idAuthentication = req->idAuthentication;
+
+ *breq->preferredMessageSize = *req->preferredMessageSize;
+ *breq->maximumRecordSize = *req->maximumRecordSize;
+
+ ODR_MASK_SET(breq->options, Z_Options_search);
+ ODR_MASK_SET(breq->options, Z_Options_present);
+ ODR_MASK_SET(breq->options, Z_Options_namedResultSets);
+ ODR_MASK_SET(breq->options, Z_Options_scan);
+
+ ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_1);
+ ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_2);
+ ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_3);
+
b->m_package->request() = init_apdu;
b->m_package->copy_filter(package);
ODR_MASK_SET(f_resp->options, Z_Options_search);
ODR_MASK_SET(f_resp->options, Z_Options_present);
ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets);
-
+ ODR_MASK_SET(f_resp->options, Z_Options_scan);
+
ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1);
ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2);
ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3);
int no_failed = 0;
int no_succeeded = 0;
+
+ Odr_int preferredMessageSize = *req->preferredMessageSize;
+ Odr_int maximumRecordSize = *req->maximumRecordSize;
for (bit = m_backend_list.begin(); bit != m_backend_list.end(); )
{
PackagePtr p = (*bit)->m_package;
-
+
if (p->session().is_closed())
{
// failed. Remove from list and increment number of failed
if (!ODR_MASK_GET(b_resp->protocolVersion, i))
ODR_MASK_CLEAR(f_resp->protocolVersion, i);
if (*b_resp->result)
+ {
no_succeeded++;
+ if (preferredMessageSize > *b_resp->preferredMessageSize)
+ preferredMessageSize = *b_resp->preferredMessageSize;
+ if (maximumRecordSize > *b_resp->maximumRecordSize)
+ maximumRecordSize = *b_resp->maximumRecordSize;
+ }
else
+ {
+ if (!f_resp->userInformationField
+ && b_resp->userInformationField)
+ f_resp->userInformationField = b_resp->userInformationField;
no_failed++;
+ }
}
else
no_failed++;
bit++;
}
+ *f_resp->preferredMessageSize = preferredMessageSize;
+ *f_resp->maximumRecordSize = maximumRecordSize;
+
if (m_p->m_hide_unavailable)
{
if (no_succeeded == 0)
void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
{
- // create search request
+ // create search request
Z_SearchRequest *req = apdu_req->u.searchRequest;
// save these for later
- int smallSetUpperBound = *req->smallSetUpperBound;
- int largeSetLowerBound = *req->largeSetLowerBound;
- int mediumSetPresentNumber = *req->mediumSetPresentNumber;
-
+ Odr_int smallSetUpperBound = *req->smallSetUpperBound;
+ Odr_int largeSetLowerBound = *req->largeSetLowerBound;
+ Odr_int mediumSetPresentNumber = *req->mediumSetPresentNumber;
+
// they are altered now - to disable piggyback
*req->smallSetUpperBound = 0;
*req->largeSetLowerBound = 1;
- *req->mediumSetPresentNumber = 1;
+ *req->mediumSetPresentNumber = 0;
int default_num_db = req->num_databaseNames;
char **default_db = req->databaseNames;
{
PackagePtr p = (*bit)->m_package;
mp::odr odr;
-
+
if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
&req->num_databaseNames,
&req->databaseNames))
// look at each response
FrontendSet resultSet(std::string(req->resultSetName));
- int result_set_size = 0;
- Z_Records *z_records_diag = 0; // no diagnostics (yet)
+ mp::odr odr;
+ Odr_int result_set_size = 0;
+ Z_DiagRecs *z_diag = 0;
+ int no_successful = 0;
+ PackagePtr close_p;
+
for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
{
PackagePtr p = (*bit)->m_package;
-
- if (p->session().is_closed()) // if any backend closes, close frontend
- package.session().close();
-
+
+ // save closing package for at least one target
+ if (p->session().is_closed())
+ close_p = p;
+
Z_GDU *gdu = p->response().get();
if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
Z_APDU_searchResponse)
{
Z_APDU *b_apdu = gdu->u.z3950;
Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
-
+
// see we get any errors (AKA diagnstics)
if (b_resp->records)
{
- if (b_resp->records->which == Z_Records_NSD
- || b_resp->records->which == Z_Records_multipleNSD)
- z_records_diag = b_resp->records;
- // we may set this multiple times (TOO BAD!)
+ if (b_resp->records->which == Z_Records_NSD)
+ {
+ if (!z_diag)
+ {
+ z_diag = (Z_DiagRecs *)
+ odr_malloc(odr, sizeof(*z_diag));
+ z_diag->num_diagRecs = 0;
+ z_diag->diagRecs = (Z_DiagRec**)
+ odr_malloc(odr, sizeof(*z_diag->diagRecs));
+ }
+ else
+ {
+ Z_DiagRec **n = (Z_DiagRec **)
+ odr_malloc(odr,
+ (1+z_diag->num_diagRecs) * sizeof(*n));
+ memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs
+ * sizeof(*n));
+ z_diag->diagRecs = n;
+ }
+ Z_DiagRec *nr = (Z_DiagRec *) odr_malloc(odr, sizeof(*nr));
+ nr->which = Z_DiagRec_defaultFormat;
+ nr->u.defaultFormat =
+ b_resp->records->u.nonSurrogateDiagnostic;
+ z_diag->diagRecs[z_diag->num_diagRecs++] = nr;
+ }
+ else if (b_resp->records->which == Z_Records_multipleNSD)
+ {
+ // we may set this multiple times (TOO BAD!)
+ z_diag = b_resp->records->u.multipleNonSurDiagnostics;
+ }
+ else
+ no_successful++; // probably piggyback
}
+ else
+ no_successful++; // no records and no diagnostics
BackendSet backendSet;
backendSet.m_backend = *bit;
backendSet.m_count = *b_resp->resultCount;
result_set_size += *b_resp->resultCount;
resultSet.m_backend_sets.push_back(backendSet);
}
- else
- {
- // if any target does not return search response - return that
- package.response() = p->response();
- return;
- }
}
- mp::odr odr;
Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
+ yaz_log(YLOG_LOG, "no_successful=%d is_closed=%s hide_errors=%s",
+ no_successful,
+ close_p ? "true" : "false",
+ m_p->m_hide_errors ? "true" : "false");
*f_resp->resultCount = result_set_size;
- if (z_records_diag)
+ if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
{
- // search error
- f_resp->records = z_records_diag;
- package.response() = f_apdu;
+ package.session().close();
+ package.response() = close_p->response();
return;
}
+ if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
+ {
+ f_resp->records = (Z_Records *)
+ odr_malloc(odr, sizeof(*f_resp->records));
+ if (z_diag->num_diagRecs > 1)
+ {
+ f_resp->records->which = Z_Records_multipleNSD;
+ f_resp->records->u.multipleNonSurDiagnostics = z_diag;
+ }
+ else
+ {
+ f_resp->records->which = Z_Records_NSD;
+ f_resp->records->u.nonSurrogateDiagnostic =
+ z_diag->diagRecs[0]->u.defaultFormat;
+ }
+ }
// assume OK
m_sets[resultSet.m_setname] = resultSet;
- int number;
+ Odr_int number;
mp::util::piggyback(smallSetUpperBound,
- largeSetLowerBound,
- mediumSetPresentNumber,
- result_set_size,
- number);
+ largeSetLowerBound,
+ mediumSetPresentNumber,
+ 0, 0,
+ result_set_size,
+ number, 0);
Package pp(package.session(), package.origin());
- if (number > 0)
+ if (z_diag == 0 && number > 0)
{
pp.copy_filter(package);
Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
*p_req->numberOfRecordsRequested = number;
pp.request() = p_apdu;
present(pp, p_apdu);
-
+
if (pp.session().is_closed())
package.session().close();
-
+
Z_GDU *gdu = pp.response().get();
if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
Z_APDU_presentResponse)
{
Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
f_resp->records = p_res->records;
- *f_resp->numberOfRecordsReturned =
+ *f_resp->numberOfRecordsReturned =
*p_res->numberOfRecordsReturned;
- *f_resp->nextResultSetPosition =
+ *f_resp->nextResultSetPosition =
*p_res->nextResultSetPosition;
}
- else
+ else
{
- package.response() = pp.response();
+ package.response() = pp.response();
return;
}
}
void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
{
- // create present request
+ // create present request
Z_PresentRequest *req = apdu_req->u.presentRequest;
Sets_it it;
if (it == m_sets.end())
{
mp::odr odr;
- Z_APDU *apdu =
+ Z_APDU *apdu =
odr.create_presentResponse(
apdu_req,
YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
*req->resultSetStartPoint = start;
*req->numberOfRecordsRequested = end - start + 1;
-
+
p->request() = apdu_req;
p->copy_filter(package);
for (; pbit != present_backend_list.end(); pbit++)
{
PackagePtr p = (*pbit)->m_package;
-
+
if (p->session().is_closed()) // if any backend closes, close frontend
package.session().close();
-
+
Z_GDU *gdu = p->response().get();
if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
Z_APDU_presentResponse)
{
Z_APDU *b_apdu = gdu->u.z3950;
Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
-
+
// see we get any errors (AKA diagnstics)
if (b_resp->records)
{
}
else
{
- // if any target does not return present response - return that
+ // if any target does not return present response - return that
package.response() = p->response();
return;
}
}
else if (number < 0 || (size_t) number > jobs.size())
{
- f_apdu =
+ f_apdu =
odr.create_presentResponse(
apdu_req,
YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
{
PackagePtr p = jit->m_backend->m_package;
-
+
Z_GDU *gdu = p->response().get();
Z_APDU *b_apdu = gdu->u.z3950;
Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
nprl->records[i] = (Z_NamePlusRecord*)
odr_malloc(odr, sizeof(Z_NamePlusRecord));
int inside_pos = jit->m_pos - jit->m_start;
- if (inside_pos >= b_resp->records->
+ if (!b_resp->records || inside_pos >= b_resp->records->
u.databaseOrSurDiagnostics->num_records)
break;
*nprl->records[i] = *b_resp->records->
if (m_backend_list.size() > 1)
{
mp::odr odr;
- Z_APDU *f_apdu =
+ Z_APDU *f_apdu =
odr.create_scanResponse(
apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
package.response() = f_apdu;
{
PackagePtr p = (*bit)->m_package;
mp::odr odr;
-
+
if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
&req->num_databaseNames,
&req->databaseNames))
for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
{
PackagePtr p = (*bit)->m_package;
-
+
if (p->session().is_closed()) // if any backend closes, close frontend
package.session().close();
-
+
Z_GDU *gdu = p->response().get();
if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
Z_APDU_scanResponse)
}
else
{
- // if any target does not return scan response - return that
+ // if any target does not return scan response - return that
package.response() = p->response();
return;
}
return e;
}
+void yf::Multi::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
+{
+ std::list<BackendPtr>::const_iterator bit;
+ for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+ {
+ PackagePtr p = (*bit)->m_package;
+ mp::odr odr;
+
+ p->request() = apdu_req;
+ p->copy_filter(package);
+ }
+ multi_move(m_backend_list);
+ for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+ {
+ PackagePtr p = (*bit)->m_package;
+
+ if (p->session().is_closed()) // if any backend closes, close frontend
+ package.session().close();
+
+ package.response() = p->response();
+ }
+}
+
+
void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
{
Z_ScanRequest *req = apdu_req->u.scanRequest;
{
PackagePtr p = (*bit)->m_package;
mp::odr odr;
-
+
if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
&req->num_databaseNames,
&req->databaseNames))
for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
{
PackagePtr p = (*bit)->m_package;
-
+
if (p->session().is_closed()) // if any backend closes, close frontend
package.session().close();
-
+
Z_GDU *gdu = p->response().get();
if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
Z_APDU_scanResponse)
Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
Z_ScanResponse *f_res = f_apdu->u.scanResponse;
- f_res->entries->nonsurrogateDiagnostics =
+ f_res->entries->nonsurrogateDiagnostics =
res->entries->nonsurrogateDiagnostics;
- f_res->entries->num_nonsurrogateDiagnostics =
+ f_res->entries->num_nonsurrogateDiagnostics =
res->entries->num_nonsurrogateDiagnostics;
package.response() = f_apdu;
}
if (my.m_norm_term.length())
{
- ScanTermInfoList::iterator it =
+ ScanTermInfoList::iterator it =
entries_before.begin();
while (it != entries_before.end() && my <*it)
it++;
- if (my == *it)
+ if (it != entries_before.end() && my == *it)
{
it->m_count += my.m_count;
}
}
if (my.m_norm_term.length())
{
- ScanTermInfoList::iterator it =
+ ScanTermInfoList::iterator it =
entries_after.begin();
while (it != entries_after.end() && *it < my)
it++;
- if (my == *it)
+ if (it != entries_after.end() && my == *it)
{
it->m_count += my.m_count;
}
}
}
- }
+ }
}
else
{
- // if any target does not return scan response - return that
+ // if any target does not return scan response - return that
package.response() = p->response();
return;
}
{
std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
}
-
+
std::cout << "AFTER\n";
it = entries_after.begin();
for(; it != entries_after.end(); it++)
mp::odr odr;
Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
Z_ScanResponse *resp = f_apdu->u.scanResponse;
-
+
int number_returned = *req->numberOfTermsRequested;
int position_returned = *req->preferredPositionInResponse;
-
+
resp->entries->num_entries = number_returned;
resp->entries->entries = (Z_Entry**)
odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
FrontendPtr f = m_p->get_frontend(package);
Z_GDU *gdu = package.request().get();
-
+
if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
Z_APDU_initRequest && !f->m_is_multi)
{
if (apdu->which == Z_APDU_initRequest)
{
mp::odr odr;
-
+
package.response() = odr.create_close(
apdu,
Z_Close_protocolError,
"double init");
-
+
package.session().close();
}
else if (apdu->which == Z_APDU_searchRequest)
{
f->scan2(package, apdu);
}
+ else if (apdu->which == Z_APDU_close)
+ {
+ f->relay_apdu(package, apdu);
+ }
else
{
mp::odr odr;
-
+
package.response() = odr.create_close(
apdu, Z_Close_protocolError,
"unsupported APDU in filter multi");
-
+
package.session().close();
}
}
m_p->release_frontend(package);
}
-void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only)
+void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only,
+ const char *path)
{
for (ptr = ptr->children; ptr; ptr = ptr->next)
{
continue;
if (!strcmp((const char *) ptr->name, "target"))
{
- std::string route = mp::xml::get_route(ptr);
+ std::string auth;
+ std::string route = mp::xml::get_route(ptr, auth);
std::string target = mp::xml::get_text(ptr);
- m_p->m_route_patterns.push_back(Multi::Map(target, route));
+ if (target.length() == 0)
+ target = route;
+ m_p->m_route_patterns.push_back(Multi::Map(target, route, auth));
}
else if (!strcmp((const char *) ptr->name, "hideunavailable"))
{
m_p->m_hide_unavailable = true;
}
+ else if (!strcmp((const char *) ptr->name, "hideerrors"))
+ {
+ m_p->m_hide_errors = true;
+ }
else if (!strcmp((const char *) ptr->name, "mergetype"))
{
std::string mergetype = mp::xml::get_text(ptr);
else
{
throw mp::filter::FilterException
- ("Bad element "
+ ("Bad element "
+ std::string((const char *) ptr->name)
+ " in multi filter");
}