X-Git-Url: http://lists.indexdata.com/cgi-bin?a=blobdiff_plain;f=src%2Ffilter_multi.cpp;h=fdfbe89469a36dbbb8725cc3897a75bf0154bc0a;hb=81038fce7a37619ed36a6f84cb0bec7d0d4f3789;hp=6cea8f80ccf24f88e17db36b127460255d880eee;hpb=60ce47a7101062380a2d05e0433b7d74dd3c1711;p=metaproxy-moved-to-github.git

diff --git a/src/filter_multi.cpp b/src/filter_multi.cpp
index 6cea8f8..fdfbe89 100644
--- a/src/filter_multi.cpp
+++ b/src/filter_multi.cpp
@@ -1,9 +1,26 @@
-/* $Id: filter_multi.cpp,v 1.18 2006-05-15 13:53:37 adam Exp $
-   Copyright (c) 2005-2006, Index Data.
+/* $Id: filter_multi.cpp,v 1.30 2008-01-19 09:02:11 adam Exp $
+   Copyright (c) 2005-2007, Index Data.
 
-%LICENSE%
+This file is part of Metaproxy.
+
+Metaproxy is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Metaproxy; see the file LICENSE.  If not, write to the
+Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+02111-1307, USA.
  */
 
+#include <yaz/log.h>
+
 #include "config.hpp"
 
 #include "filter.hpp"
@@ -31,7 +48,10 @@ namespace yf = mp::filter;
 
 namespace metaproxy_1 {
     namespace filter {
-
+        enum multi_merge_type {
+            round_robin,
+            serve_order
+        };
         struct Multi::BackendSet {
             BackendPtr m_backend;
             int m_count;
@@ -47,16 +67,20 @@ namespace metaproxy_1 {
             Z_Entry *get_entry(ODR odr);
         };
         struct Multi::FrontendSet {
-            struct PresentJob {
+            class PresentJob {
+            public:
                 BackendPtr m_backend;
-                int m_pos;
-                int m_inside_pos;
+                int m_pos; // position for backend (1=first, 2=second,..
+                int m_start; // present request start
+                PresentJob(BackendPtr ptr, int pos) : 
+                    m_backend(ptr), m_pos(pos), m_start(0) {};
             };
             FrontendSet(std::string setname);
             FrontendSet();
             ~FrontendSet();
 
             void round_robin(int pos, int number, std::list<PresentJob> &job);
+            void serve_order(int pos, int number, std::list<PresentJob> &job);
 
             std::list<BackendSet> m_backend_sets;
             std::string m_setname;
@@ -104,6 +128,7 @@ namespace metaproxy_1 {
             boost::condition m_cond_session_ready;
             std::map<mp::Session, FrontendPtr> m_clients;
             bool m_hide_unavailable;
+            multi_merge_type m_merge_type;
         };
     }
 }
@@ -111,6 +136,7 @@ namespace metaproxy_1 {
 yf::Multi::Rep::Rep()
 {
     m_hide_unavailable = false;
+    m_merge_type = round_robin;
 }
 
 bool yf::Multi::BackendSet::operator < (const BackendSet &k) const
@@ -128,7 +154,7 @@ yf::Multi::Frontend::~Frontend()
 {
 }
 
-yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(Package &package)
+yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package)
 {
     boost::mutex::scoped_lock lock(m_mutex);
 
@@ -153,7 +179,7 @@ yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(Package &package)
     return f;
 }
 
-void yf::Multi::Rep::release_frontend(Package &package)
+void yf::Multi::Rep::release_frontend(mp::Package &package)
 {
     boost::mutex::scoped_lock lock(m_mutex);
     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
@@ -212,7 +238,7 @@ void yf::Multi::Backend::operator() (void)
 }
 
 
-void yf::Multi::Frontend::close(Package &package)
+void yf::Multi::Frontend::close(mp::Package &package)
 {
     std::list<BackendPtr>::const_iterator bit;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
@@ -237,16 +263,37 @@ void yf::Multi::Frontend::multi_move(std::list<BackendPtr> &blist)
     g.join_all();
 }
 
+void yf::Multi::FrontendSet::serve_order(int start, int number,
+                                         std::list<PresentJob> &jobs)
+{
+    int i;
+    for (i = 0; i < number; i++)
+    {
+        std::list<BackendSet>::const_iterator bsit;
+        int voffset = 0;
+        int offset = start + i - 1;
+        for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); 
+             bsit++)
+        {
+            if (offset >= voffset && offset < voffset + bsit->m_count)
+            {
+                PresentJob job(bsit->m_backend, offset - voffset + 1);
+                jobs.push_back(job);
+                break;
+            }
+            voffset += bsit->m_count;
+        }
+    }
+}
+
 void yf::Multi::FrontendSet::round_robin(int start, int number,
                                          std::list<PresentJob> &jobs)
 {
     std::list<int> pos;
-    std::list<int> inside_pos;
     std::list<BackendSet>::const_iterator bsit;
     for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
     {
         pos.push_back(1);
-        inside_pos.push_back(0);
     }
 
     int p = 1;
@@ -285,8 +332,6 @@ void yf::Multi::FrontendSet::round_robin(int start, int number,
         // skip on each set.. before "present range"..
         p = p + skip;
         
-        std::cout << "\nSKIP min=" << min << " no_left=" << no_left << "\n\n";
-        
         std::list<int>::iterator psit = pos.begin();
         for (psit = pos.begin(); psit != pos.end(); psit++)
             *psit += min;
@@ -300,10 +345,9 @@ void yf::Multi::FrontendSet::round_robin(int start, int number,
     {
         more = false;
         std::list<int>::iterator psit = pos.begin();
-        std::list<int>::iterator esit = inside_pos.begin();
         bsit = m_backend_sets.begin();
 
-        for (; bsit != m_backend_sets.end(); psit++,esit++,bsit++)
+        for (; bsit != m_backend_sets.end(); psit++,bsit++)
         {
             if (fetched >= number)
             {
@@ -314,12 +358,8 @@ void yf::Multi::FrontendSet::round_robin(int start, int number,
             {
                 if (p >= start)
                 {
-                    PresentJob job;
-                    job.m_backend = bsit->m_backend;
-                    job.m_pos = *psit;
-                    job.m_inside_pos = *esit;
+                    PresentJob job(bsit->m_backend, *psit);
                     jobs.push_back(job);
-                    (*esit)++;
                     fetched++;
                 }
                 (*psit)++;
@@ -330,13 +370,13 @@ void yf::Multi::FrontendSet::round_robin(int start, int number,
     }
 }
 
-void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu)
+void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
 {
     Z_InitRequest *req = gdu->u.z3950->u.initRequest;
 
     std::list<std::string> targets;
 
-    mp::util::get_vhost_otherinfo(&req->otherInfo, false, targets);
+    mp::util::get_vhost_otherinfo(req->otherInfo, targets);
 
     if (targets.size() < 1)
     {
@@ -409,8 +449,9 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu)
     {
         PackagePtr p = (*bit)->m_package;
         
-        if (p->session().is_closed()) // if any backend closes, close frontend
+        if (p->session().is_closed())
         {
+            // failed. Remove from list and increment number of failed
             no_failed++;
             bit = m_backend_list.erase(bit);
             continue;
@@ -461,7 +502,7 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu)
     package.response() = f_apdu;
 }
 
-void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
 {
     // create search request 
     Z_SearchRequest *req = apdu_req->u.searchRequest;
@@ -595,7 +636,7 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req)
     package.response() = f_apdu; // in this scope because of p
 }
 
-void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
 {
     // create present request 
     Z_PresentRequest *req = apdu_req->u.presentRequest;
@@ -616,7 +657,20 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req)
     std::list<Multi::FrontendSet::PresentJob> jobs;
     int start = *req->resultSetStartPoint;
     int number = *req->numberOfRecordsRequested;
-    it->second.round_robin(start, number, jobs);
+
+    if (m_p->m_merge_type == round_robin)
+        it->second.round_robin(start, number, jobs);
+    else if (m_p->m_merge_type == serve_order)
+        it->second.serve_order(start, number, jobs);
+
+    if (0)
+    {
+        std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
+        for (jit = jobs.begin(); jit != jobs.end(); jit++)
+        {
+            yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos);
+        }
+    }
 
     std::list<BackendPtr> present_backend_list;
 
@@ -624,22 +678,33 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req)
     bsit = it->second.m_backend_sets.begin();
     for (; bsit != it->second.m_backend_sets.end(); bsit++)
     {
-        std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
         int start = -1;
         int end = -1;
-        
-        for (jit = jobs.begin(); jit != jobs.end(); jit++)
         {
-            if (jit->m_backend == bsit->m_backend)
+            std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
+            for (jit = jobs.begin(); jit != jobs.end(); jit++)
             {
-                if (start == -1 || jit->m_pos < start)
-                    start = jit->m_pos;
-                if (end == -1 || jit->m_pos > end)
-                    end = jit->m_pos;
+                if (jit->m_backend == bsit->m_backend)
+                {
+                    if (start == -1 || jit->m_pos < start)
+                        start = jit->m_pos;
+                    if (end == -1 || jit->m_pos > end)
+                        end = jit->m_pos;
+                }
             }
         }
         if (start != -1)
         {
+            std::list<Multi::FrontendSet::PresentJob>::iterator jit;
+            for (jit = jobs.begin(); jit != jobs.end(); jit++)
+            {
+                if (jit->m_backend == bsit->m_backend)
+                {
+                    if (jit->m_pos >= start && jit->m_pos <= end)
+                        jit->m_start = start;
+                }
+            }
+
             PackagePtr p = bsit->m_backend->m_package;
 
             *req->resultSetStartPoint = start;
@@ -710,7 +775,7 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req)
             odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
         int i = 0;
         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
-        for (jit = jobs.begin(); jit != jobs.end(); jit++)
+        for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
         {
             PackagePtr p = jit->m_backend->m_package;
             
@@ -718,17 +783,25 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req)
             Z_APDU *b_apdu = gdu->u.z3950;
             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
 
-            nprl->records[i++] =
-                b_resp->records->u.databaseOrSurDiagnostics->
-                records[jit->m_inside_pos];
+            nprl->records[i] = (Z_NamePlusRecord*)
+                odr_malloc(odr, sizeof(Z_NamePlusRecord));
+            int inside_pos = jit->m_pos - jit->m_start;
+            if (inside_pos >= b_resp->records->
+                u.databaseOrSurDiagnostics->num_records)
+                break;
+	    *nprl->records[i] = *b_resp->records->
+                u.databaseOrSurDiagnostics->records[inside_pos];
+            nprl->records[i]->databaseName =
+                    odr_strdup(odr, jit->m_backend->m_vhost.c_str());
         }
+        nprl->num_records = i; // usually same as jobs.size();
         *f_resp->nextResultSetPosition = start + i;
         *f_resp->numberOfRecordsReturned = i;
     }
     package.response() = f_apdu;
 }
 
-void yf::Multi::Frontend::scan1(Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req)
 {
     if (m_backend_list.size() > 1)
     {
@@ -819,7 +892,7 @@ Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
     return e;
 }
 
-void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
 {
     Z_ScanRequest *req = apdu_req->u.scanRequest;
 
@@ -978,7 +1051,7 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req)
         }
     }
 
-    if (true)
+    if (false)
     {
         std::cout << "BEFORE\n";
         ScanTermInfoList::iterator it = entries_before.begin();
@@ -1047,7 +1120,7 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req)
 }
 
 
-void yf::Multi::process(Package &package) const
+void yf::Multi::process(mp::Package &package) const
 {
     FrontendPtr f = m_p->get_frontend(package);
 
@@ -1117,12 +1190,24 @@ void mp::filter::Multi::configure(const xmlNode * ptr)
         {
             m_p->m_hide_unavailable = true;
         }
+        else if (!strcmp((const char *) ptr->name, "mergetype"))
+        {
+            std::string mergetype = mp::xml::get_text(ptr);
+            if (mergetype == "roundrobin")
+                m_p->m_merge_type = round_robin;
+            else if (mergetype == "serveorder")
+                m_p->m_merge_type = serve_order;
+            else
+                throw mp::filter::FilterException
+                    ("Bad mergetype "  + mergetype + " in multi filter");
+
+        }
         else
         {
             throw mp::filter::FilterException
                 ("Bad element " 
                  + std::string((const char *) ptr->name)
-                 + " in virt_db filter");
+                 + " in multi filter");
         }
     }
 }