From: Sebastian Hammer <quinn@indexdata.com>
Date: Wed, 3 Jan 2007 06:23:44 +0000 (+0000)
Subject: Added XSLT support. Configuration file (e.g. etc/pazpar2.cfg) must now be specified
X-Git-Tag: before.append.child~56
X-Git-Url: http://lists.indexdata.com/cgi-bin?a=commitdiff_plain;h=dde4a5914bbb5511a91e73d8ab45210b48637596;p=pazpar2-moved-to-github.git

Added XSLT support. Configuration file (e.g. etc/pazpar2.cfg) must now be specified
by option -f before targets are loaded with -s (this is temporary). Listener portno
and proxy is still controlled from command line for now! Incoming records are
normalized to intermediate format with XSLT. Added support for multiple facets --
generated dynamically from the XSLT normalization step. marc21.xsl is for MARC.
---

diff --git a/PROTOCOL b/PROTOCOL
index 35d1a00..8ae4a9d 100644
--- a/PROTOCOL
+++ b/PROTOCOL
@@ -117,3 +117,37 @@ Output:
     </title>
   </hit>
 </show>
+
+termlist
+
+Retrieves term list(s)
+
+parameters:
+
+session
+name       -- comma-separated list of termlist names (default "subject")
+
+Example:
+
+search.pz2?session=2044502273&command=termlist&name=author,subject
+
+Output:
+
+<termlist>
+  <list name="author">
+    <term>
+      <name>Donald Knuth</name>
+      <frequency>10</frequency>
+    </term>
+    <term>
+      <name>Robert Pirsig</name>
+      <frequency>2</frequency>
+    </term>
+  </list>
+  <list name="subject">
+    <term>
+      <name>Computer programming</name>
+      <frequency>10</frequency>
+    </term>
+  </list>
+</termlist>
diff --git a/etc/marc21.xsl b/etc/marc21.xsl
index 24d0e2c..bcb30e7 100644
--- a/etc/marc21.xsl
+++ b/etc/marc21.xsl
@@ -8,18 +8,20 @@
   <xsl:template match="/marc:record">
     <pz:record>
 
-      <pz:metadata type="title">
-	<xsl:value-of select="marc:datafield[@tag='245']/marc:subfield[@code='a']"/>
-	<xsl:value-of select="marc:datafield[@tag='245']/marc:subfield[@code='b']"/>
-      </pz:metadata>
-
-      <pz:mergekey>
+      <xsl:attribute name="mergekey">
         <xsl:text>title </xsl:text>
 	<xsl:value-of select="marc:datafield[@tag='245']/marc:subfield[@code='a']"/>
 	<xsl:value-of select="marc:datafield[@tag='245']/marc:subfield[@code='b']"/>
 	<xsl:text> author </xsl:text>
 	<xsl:value-of select="marc:datafield[@tag='100']/marc:subfield[@code='a']"/>
-      </pz:mergekey>
+      </xsl:attribute>
+
+
+      <pz:metadata type="title">
+	<xsl:value-of select="marc:datafield[@tag='245']/marc:subfield[@code='a']"/>
+	<xsl:text> </xsl:text>
+	<xsl:value-of select="marc:datafield[@tag='245']/marc:subfield[@code='b']"/>
+      </pz:metadata>
 
       <xsl:for-each select="marc:datafield[@tag='650']">
 	<pz:facet type="subject">
diff --git a/etc/pazpar2.cfg b/etc/pazpar2.cfg
index a640319..e63a98c 100644
--- a/etc/pazpar2.cfg
+++ b/etc/pazpar2.cfg
@@ -1,5 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<pazpar2 xmlns="http://www.indexdata.com/pazpar2/1.0">
+<?xml version="1.0" encoding="UTF-8"?> <pazpar2 xmlns="http://www.indexdata.com/pazpar2/1.0">
 
 <!-- NOTE: This is not yet a functional config file. It's meant as a scratchpad/
      discussion area to figure out just how the heck to structure this kind of
@@ -35,8 +34,7 @@
 
 <retrievalprofile>
   <requestsyntax>marc21</requestsyntax>
-  <charset>marc-8</charset>
-  <map type="iso2709" charset="marc-8" format="marcxml"/>
+  <nativesyntax name="iso2709" format="marc21" encoding="marc-8" mapto="marcxml"/>
   <map type="xslt" stylesheet="marc21.xsl"/>
 </retrievalprofile>
 
diff --git a/src/Makefile b/src/Makefile
index dc2ae88..04f9023 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -1,6 +1,6 @@
 # ParaZ. Copyright (C) 2000-2004, Index Data ApS
 # All rights reserved.
-# $Id: Makefile,v 1.3 2006-12-27 21:11:10 quinn Exp $
+# $Id: Makefile,v 1.4 2007-01-03 06:23:44 quinn Exp $
 
 SHELL=/bin/sh
 
@@ -17,7 +17,8 @@ PROGO=pazpar2.o eventl.o util.o command.o http.o http_command.o termlists.o \
 all: $(PROG)
 
 $(PROG): $(PROGO)
-	$(CC) $(CFLAGS) $(YAZCFLAGS) -o $(PROG) $(PROGO) $(YAZLIBS)
+	$(CC) $(CFLAGS) $(YAZCFLAGS) -o $(PROG) $(PROGO) yaz/src/.libs/libyaz.a $(YAZLIBS)
+#	$(CC) $(CFLAGS) $(YAZCFLAGS) -o $(PROG) $(PROGO) $(YAZLIBS)
 
 .c.o:
 	$(CC) -c $(CFLAGS) -I. $(YAZCFLAGS) $<
@@ -29,18 +30,18 @@ clean:
 ## Dependencies go below
 
 command.o: command.c command.h util.h eventl.h pazpar2.h termlists.h \
-  relevance.h reclists.h
+  relevance.h reclists.h config.h
 config.o: config.c config.h
 eventl.o: eventl.c eventl.h
 http.o: http.c command.h util.h eventl.h pazpar2.h termlists.h \
-  relevance.h reclists.h http.h http_command.h
+  relevance.h reclists.h config.h http.h http_command.h
 http_command.o: http_command.c command.h util.h eventl.h pazpar2.h \
-  termlists.h relevance.h reclists.h http.h http_command.h
+  termlists.h relevance.h reclists.h config.h http.h http_command.h
 pazpar2.o: pazpar2.c pazpar2.h termlists.h relevance.h reclists.h \
-  eventl.h command.h http.h config.h
+  eventl.h config.h command.h http.h
 reclists.o: reclists.c pazpar2.h termlists.h relevance.h reclists.h \
-  eventl.h
+  eventl.h config.h
 relevance.o: relevance.c relevance.h pazpar2.h termlists.h eventl.h \
-  reclists.h
+  config.h reclists.h
 termlists.o: termlists.c termlists.h
 util.o: util.c
diff --git a/src/config.c b/src/config.c
index e02c670..3f02149 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1,9 +1,12 @@
-/* $Id: config.c,v 1.2 2006-12-27 21:11:10 quinn Exp $ */
+/* $Id: config.c,v 1.3 2007-01-03 06:23:44 quinn Exp $ */
 
 #include <string.h>
 
 #include <libxml/parser.h>
 #include <libxml/tree.h>
+#include <libxslt/xslt.h>
+#include <libxslt/transform.h>
+#include <libxslt/xsltutils.h>
 
 #include <yaz/yaz-util.h>
 #include <yaz/nmem.h>
@@ -12,9 +15,13 @@
 #include "config.h"
 
 static NMEM nmem = 0;
+static char confdir[256] = ".";
 
 struct conf_config *config = 0;
 
+/* Code to parse configuration file */
+/* ==================================================== */
+
 static struct conf_service *parse_service(xmlNode *node)
 {
     xmlNode *n;
@@ -98,10 +105,147 @@ static struct conf_server *parse_server(xmlNode *node)
     return r;
 }
 
+static xsltStylesheet *load_stylesheet(const char *fname)
+{
+    char path[256];
+    sprintf(path, "%s/%s", confdir, fname);
+    return xsltParseStylesheetFile(path);
+}
+
+static void setup_marc(struct conf_retrievalprofile *r)
+{
+    yaz_iconv_t cm;
+    r->yaz_marc = yaz_marc_create();
+    if (!(cm = yaz_iconv_open("utf-8", r->native_encoding)))
+    {
+        yaz_log(YLOG_WARN, "Unable to support mapping from %s", r->native_encoding);
+        return;
+    }
+    yaz_marc_iconv(r->yaz_marc, cm);
+}
+
+static struct conf_retrievalprofile *parse_retrievalprofile(xmlNode *node)
+{
+    struct conf_retrievalprofile *r = nmem_malloc(nmem, sizeof(struct conf_retrievalprofile));
+    xmlNode *n;
+    struct conf_retrievalmap **rm = &r->maplist;
+
+    r->requestsyntax = 0;
+    r->native_syntax = Nativesyn_xml;
+    r->native_format = Nativeform_na;
+    r->native_encoding = 0;
+    r->native_mapto = Nativemapto_na;
+    r->yaz_marc = 0;
+    r->maplist = 0;
+    r->next = 0;
+
+    for (n = node->children; n; n = n->next)
+    {
+        if (n->type != XML_ELEMENT_NODE)
+            continue;
+        if (!strcmp(n->name, "requestsyntax"))
+        {
+            xmlChar *content = xmlNodeGetContent(n);
+            if (content)
+                r->requestsyntax = nmem_strdup(nmem, content);
+        }
+        else if (!strcmp(n->name, "nativesyntax"))
+        {
+            xmlChar *name = xmlGetProp(n, "name");
+            xmlChar *format = xmlGetProp(n, "format");
+            xmlChar *encoding = xmlGetProp(n, "encoding");
+            xmlChar *mapto = xmlGetProp(n, "mapto");
+            if (!name)
+            {
+                yaz_log(YLOG_WARN, "Missing name in 'nativesyntax' element");
+                return 0;
+            }
+            if (!strcmp(name, "iso2709"))
+            {
+                r->native_syntax = Nativesyn_iso2709;
+                // Set a few defaults, too
+                r->native_format = Nativeform_marc21;
+                r->native_mapto = Nativemapto_marcxml;
+                r->native_encoding = "marc-8";
+                setup_marc(r);
+            }
+            else if (!strcmp(name, "xml"))
+                r->native_syntax = Nativesyn_xml;
+            else
+            {
+                yaz_log(YLOG_WARN, "Unknown native syntax name %s", name);
+                return 0;
+            }
+            if (format)
+            {
+                if (!strcmp(format, "marc21") || !strcmp(format, "usmarc"))
+                    r->native_format = Nativeform_marc21;
+                else
+                {
+                    yaz_log(YLOG_WARN, "Unknown native format name %s", format);
+                    return 0;
+                }
+            }
+            if (encoding)
+                r->native_encoding = encoding;
+            if (mapto)
+            {
+                if (!strcmp(mapto, "marcxml"))
+                    r->native_mapto = Nativemapto_marcxml;
+                else if (!strcmp(mapto, "marcxchange"))
+                    r->native_mapto = Nativemapto_marcxchange;
+                else
+                {
+                    yaz_log(YLOG_WARN, "Unknown mapto target %s", format);
+                    return 0;
+                }
+            }
+        }
+        else if (!strcmp(n->name, "map"))
+        {
+            struct conf_retrievalmap *m = nmem_malloc(nmem, sizeof(struct conf_retrievalmap));
+            xmlChar *type = xmlGetProp(n, "type");
+            xmlChar *charset = xmlGetProp(n, "charset");
+            xmlChar *format = xmlGetProp(n, "format");
+            xmlChar *stylesheet = xmlGetProp(n, "stylesheet");
+            bzero(m, sizeof(*m));
+            if (type)
+            {
+                if (!strcmp(type, "xslt"))
+                    m->type = Map_xslt;
+                else
+                {
+                    yaz_log(YLOG_WARN, "Unknown map type: %s", type);
+                    return 0;
+                }
+            }
+            if (charset)
+                m->charset = nmem_strdup(nmem, charset);
+            if (format)
+                m->format = nmem_strdup(nmem, format);
+            if (stylesheet)
+            {
+                if (!(m->stylesheet = load_stylesheet(stylesheet)))
+                    return 0;
+            }
+            *rm = m;
+            rm = &m->next;
+        }
+        else
+        {
+            yaz_log(YLOG_FATAL, "Bad element in retrievalprofile: %s", n->name);
+            return 0;
+        }
+    }
+
+    return r;
+}
+
 static struct conf_config *parse_config(xmlNode *root)
 {
     xmlNode *n;
     struct conf_config *r = nmem_malloc(nmem, sizeof(struct conf_config));
+    struct conf_retrievalprofile **rp = &r->retrievalprofiles;
 
     r->servers = 0;
     r->queryprofiles = 0;
@@ -124,6 +268,9 @@ static struct conf_config *parse_config(xmlNode *root)
         }
         else if (!strcmp(n->name, "retrievalprofile"))
         {
+            if (!(*rp = parse_retrievalprofile(n)))
+                return 0;
+            rp = &(*rp)->next;
         }
         else
         {
@@ -137,14 +284,29 @@ static struct conf_config *parse_config(xmlNode *root)
 int read_config(const char *fname)
 {
     xmlDoc *doc = xmlReadFile(fname, NULL, 0);
-    if (!nmem)
+    const char *p;
+
+    if (!nmem)  // Initialize
+    {
         nmem = nmem_create();
+        xmlSubstituteEntitiesDefault(1);
+        xmlLoadExtDtdDefaultValue = 1;
+    }
     if (!doc)
     {
         yaz_log(YLOG_FATAL, "Failed to read %s", fname);
         exit(1);
     }
-    if ((config = parse_config(xmlDocGetRootElement(doc))))
+    if ((p = rindex(fname, '/')))
+    {
+        int len = p - fname;
+        strncpy(confdir, fname, len);
+        confdir[len] = '\0';
+    }
+    config = parse_config(xmlDocGetRootElement(doc));
+    xmlFreeDoc(doc);
+
+    if (config)
         return 1;
     else
         return 0;
diff --git a/src/config.h b/src/config.h
index 9d49ac5..53c1de6 100644
--- a/src/config.h
+++ b/src/config.h
@@ -1,6 +1,10 @@
 #ifndef CONFIG_H
 #define CONFIG_H
 
+#include <libxslt/xslt.h>
+#include <libxslt/transform.h>
+#include <libxslt/xsltutils.h>
+
 struct conf_termlist
 {
     char *name;
@@ -26,8 +30,37 @@ struct conf_queryprofile
 {
 };
 
+struct conf_retrievalmap
+{
+    enum {
+        Map_xslt
+    } type;
+    char *charset;
+    char *format;
+    xsltStylesheet *stylesheet;
+    struct conf_retrievalmap *next;
+};
+
 struct conf_retrievalprofile
 {
+    char *requestsyntax;
+    enum {
+        Nativesyn_xml,
+        Nativesyn_iso2709
+    } native_syntax;
+    enum {
+        Nativeform_na,
+        Nativeform_marc21,
+    } native_format;
+    char *native_encoding;
+    enum {
+        Nativemapto_na,
+        Nativemapto_marcxml,
+        Nativemapto_marcxchange
+    } native_mapto;
+    yaz_marc_t yaz_marc;
+    struct conf_retrievalmap *maplist;
+    struct conf_retrievalprofile *next;
 };
 
 struct conf_config
diff --git a/src/http_command.c b/src/http_command.c
index e453a9d..54a1654 100644
--- a/src/http_command.c
+++ b/src/http_command.c
@@ -1,5 +1,5 @@
 /*
- * $Id: http_command.c,v 1.2 2006-12-20 22:19:35 adam Exp $
+ * $Id: http_command.c,v 1.3 2007-01-03 06:23:44 quinn Exp $
  */
 
 #include <stdio.h>
@@ -152,21 +152,44 @@ static void cmd_termlist(struct http_channel *c)
     struct termlist_score **p;
     int len;
     int i;
+    char *name = http_argbyname(rq, "name");
 
     if (!s)
         return;
+
+    if (!name)
+        name = "subject";
+    if (strlen(name) > 255)
+        return;
+
     wrbuf_rewind(c->wrbuf);
 
     wrbuf_puts(c->wrbuf, "<termlist>");
-    p = termlist(s->psession, &len);
-    if (p)
-        for (i = 0; i < len; i++)
-        {
-            wrbuf_puts(c->wrbuf, "\n<term>");
-            wrbuf_printf(c->wrbuf, "<name>%s</name>", p[i]->term);
-            wrbuf_printf(c->wrbuf, "<frequency>%d</frequency>", p[i]->frequency);
-            wrbuf_puts(c->wrbuf, "</term>");
-        }
+    while (*name)
+    {
+        char tname[256];
+        char *tp;
+
+        if (!(tp = strchr(name, ',')))
+            tp = name + strlen(name);
+        strncpy(tname, name, tp - name);
+        tname[tp - name] = '\0';
+
+        p = termlist(s->psession, tname, &len);
+        wrbuf_printf(c->wrbuf, "\n<list name=\"%s\">\n", tname);
+        if (p)
+            for (i = 0; i < len; i++)
+            {
+                wrbuf_puts(c->wrbuf, "\n<term>");
+                wrbuf_printf(c->wrbuf, "<name>%s</name>", p[i]->term);
+                wrbuf_printf(c->wrbuf, "<frequency>%d</frequency>", p[i]->frequency);
+                wrbuf_puts(c->wrbuf, "</term>");
+            }
+        wrbuf_puts(c->wrbuf, "\n</list>");
+        name = tp;
+        if (*name == ',')
+            name++;
+    }
     wrbuf_puts(c->wrbuf, "</termlist>");
     rs->payload = nmem_strdup(rq->channel->nmem, wrbuf_buf(c->wrbuf));
     http_send_response(c);
diff --git a/src/pazpar2.c b/src/pazpar2.c
index 8605397..fd5ebf5 100644
--- a/src/pazpar2.c
+++ b/src/pazpar2.c
@@ -1,4 +1,4 @@
-/* $Id: pazpar2.c,v 1.6 2006-12-27 21:11:10 quinn Exp $ */;
+/* $Id: pazpar2.c,v 1.7 2007-01-03 06:23:44 quinn Exp $ */;
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -35,6 +35,7 @@ static void client_fatal(struct client *cl);
 static void connection_destroy(struct connection *co);
 static int client_prep_connection(struct client *cl);
 static void ingest_records(struct client *cl, Z_Records *r);
+static struct conf_retrievalprofile *database_retrieval_profile(struct database *db);
 void session_alert_watch(struct session *s, int what);
 
 IOCHAN channel_list = 0;  // Master list of connections we're handling events to
@@ -60,6 +61,7 @@ static char *client_states[] = {
 
 struct parameters global_parameters = 
 {
+    0,
     30,
     "81",
     "Index Data PazPar2 (MasterKey)",
@@ -71,7 +73,6 @@ struct parameters global_parameters =
     0,
     0,
     0,
-    0,
     0
 };
 
@@ -270,275 +271,29 @@ static void do_searchResponse(IOCHAN i, Z_APDU *a)
     }
 }
 
-const char *find_field(const char *rec, const char *field)
-{
-    char lbuf[5];
-    char *line;
-
-    lbuf[0] = '\n';
-    strcpy(lbuf + 1, field);
-
-    if ((line = strstr(rec, lbuf)))
-        return ++line;
-    else
-        return 0;
-}
-
-const char *find_subfield(const char *field, char subfield)
-{
-    const char *p = field;
-
-    while (*p && *p != '\n')
-    {
-        while (*p != '\n' && *p != '\t')
-            p++;
-        if (*p == '\t' && *(++p) == subfield) {
-            if (*(++p) == ' ')
-            {
-                while (isspace(*p))
-                    p++;
-                return p;
-            }
-        }
-    }
-    return 0;
-}
-
-// Extract 245 $a $b 100 $a
-char *extract_title(struct session *s, const char *rec)
-{
-    const char *field, *subfield;
-    char *e, *ef;
-    unsigned char *obuf, *p;
-
-    wrbuf_rewind(s->wrbuf);
-
-    if (!(field = find_field(rec, "245")))
-        return 0;
-    if (!(subfield = find_subfield(field, 'a')))
-        return 0;
-    ef = index(subfield, '\n');
-    if ((e = index(subfield, '\t')) && e < ef)
-        ef = e;
-    if (ef)
-    {
-        wrbuf_write(s->wrbuf, subfield, ef - subfield);
-        if ((subfield = find_subfield(field, 'b'))) 
-        {
-            ef = index(subfield, '\n');
-            if ((e = index(subfield, '\t')) && e < ef)
-                ef = e;
-            if (ef)
-            {
-                wrbuf_putc(s->wrbuf, ' ');
-                wrbuf_write(s->wrbuf, subfield, ef - subfield);
-            }
-        }
-    }
-    if ((field = find_field(rec, "100")))
-    {
-        if ((subfield = find_subfield(field, 'a')))
-        {
-            ef = index(subfield, '\n');
-            if ((e = index(subfield, '\t')) && e < ef)
-                ef = e;
-            if (ef)
-            {
-                wrbuf_puts(s->wrbuf, ", by ");
-                wrbuf_write(s->wrbuf, subfield, ef - subfield);
-            }
-        }
-    }
-    wrbuf_putc(s->wrbuf, '\0');
-    obuf = (unsigned char*) nmem_strdup(s->nmem, wrbuf_buf(s->wrbuf));
-    for (p = obuf; *p; p++)
-        if (*p == '&' || *p == '<' || *p > 122 || *p < ' ')
-            *p = ' ';
-    return (char*) obuf;
-}
-
-// Extract 245 $a $b 100 $a
-char *extract_mergekey(struct session *s, const char *rec)
+char *normalize_mergekey(char *buf)
 {
-    const char *field, *subfield;
-    char *e, *ef;
-    char *out, *p, *pout;
-
-    wrbuf_rewind(s->wrbuf);
-
-    if (!(field = find_field(rec, "245")))
-        return 0;
-    if (!(subfield = find_subfield(field, 'a')))
-        return 0;
-    ef = index(subfield, '\n');
-    if ((e = index(subfield, '\t')) && e < ef)
-        ef = e;
-    if (ef)
-    {
-        wrbuf_write(s->wrbuf, subfield, ef - subfield);
-        if ((subfield = find_subfield(field, 'b'))) 
-        {
-            ef = index(subfield, '\n');
-            if ((e = index(subfield, '\t')) && e < ef)
-                ef = e;
-            if (ef)
-            {
-                wrbuf_puts(s->wrbuf, " field "); 
-                wrbuf_write(s->wrbuf, subfield, ef - subfield);
-            }
-        }
-    }
-    if ((field = find_field(rec, "100")))
-    {
-        if ((subfield = find_subfield(field, 'a')))
-        {
-            ef = index(subfield, '\n');
-            if ((e = index(subfield, '\t')) && e < ef)
-                ef = e;
-            if (ef)
-            {
-                wrbuf_puts(s->wrbuf, " field "); 
-                wrbuf_write(s->wrbuf, subfield, ef - subfield);
-            }
-        }
-    }
-    wrbuf_putc(s->wrbuf, '\0');
-    p = wrbuf_buf(s->wrbuf);
-    out = pout = nmem_malloc(s->nmem, strlen(p) + 1);
+    char *p = buf, *pout = buf;
 
     while (*p)
     {
+        while (*p && !isalnum(*p))
+            p++;
         while (isalnum(*p))
             *(pout++) = tolower(*(p++));
+        if (*p)
+            *(pout++) = ' ';
         while (*p && !isalnum(*p))
             p++;
-        *(pout++) = ' ';
-    }
-    if (out != pout)
-        *(--pout) = '\0';
-
-    return out;
-}
-
-#ifdef RECHEAP
-static void push_record(struct session *s, struct record *r)
-{
-    int p;
-    assert(s->recheap_max + 1 < s->recheap_size);
-
-    s->recheap[p = ++s->recheap_max] = r;
-    while (p > 0)
-    {
-        int parent = (p - 1) >> 1;
-        if (strcmp(s->recheap[p]->merge_key, s->recheap[parent]->merge_key) < 0)
-        {
-            struct record *tmp;
-            tmp = s->recheap[parent];
-            s->recheap[parent] = s->recheap[p];
-            s->recheap[p] = tmp;
-            p = parent;
-        }
-        else
-            break;
-    }
-}
-
-static struct record *top_record(struct session *s)
-{
-    return s-> recheap_max >= 0 ?  s->recheap[0] : 0;
-}
-
-static struct record *pop_record(struct session *s)
-{
-    struct record *res;
-    int p = 0;
-    int lastnonleaf = (s->recheap_max - 1) >> 1;
-
-    if (s->recheap_max < 0)
-        return 0;
-
-    res = s->recheap[0];
-
-    s->recheap[p] = s->recheap[s->recheap_max--];
-
-    while (p <= lastnonleaf)
-    {
-        int right = (p + 1) << 1;
-        int left = right - 1;
-        int min = left;
-
-        if (right < s->recheap_max &&
-                strcmp(s->recheap[right]->merge_key, s->recheap[left]->merge_key) < 0)
-            min = right;
-        if (strcmp(s->recheap[min]->merge_key, s->recheap[p]->merge_key) < 0)
-        {
-            struct record *tmp = s->recheap[min];
-            s->recheap[min] = s->recheap[p];
-            s->recheap[p] = tmp;
-            p = min;
-        }
-        else
-            break;
     }
-    return res;
-}
+    if (buf != pout)
+        *pout = '\0';
 
-// Like pop_record but collapses identical (merge_key) records
-// The heap will contain multiple independent matching records and possibly
-// one cluster, created the last time the list was scanned
-static struct record *pop_mrecord(struct session *s)
-{
-    struct record *this;
-    struct record *next;
-
-    if (!(this = pop_record(s)))
-        return 0;
-
-    // Collapse identical records
-    while ((next = top_record(s)))
-    {
-        struct record *p, *tmpnext;
-        if (strcmp(this->merge_key, next->merge_key))
-            break;
-        // Absorb record (and clustersiblings) into a supercluster
-        for (p = next; p; p = tmpnext) {
-            tmpnext = p->next_cluster;
-            p->next_cluster = this->next_cluster;
-            this->next_cluster = p;
-        }
-
-        pop_record(s);
-    }
-    return this;
-}
-
-// Reads records in sort order. Store records in top of heapspace until rewind is called.
-static struct record *read_recheap(struct session *s)
-{
-    struct record *r = pop_mrecord(s);
-
-    if (r)
-    {
-        if (s->recheap_scratch < 0)
-            s->recheap_scratch = s->recheap_size;
-        s->recheap[--s->recheap_scratch] = r;
-    }
-
-    return r;
-}
-
-// Return records to heap after read
-static void rewind_recheap(struct session *s)
-{
-    while (s->recheap_scratch >= 0) {
-        push_record(s, s->recheap[s->recheap_scratch++]);
-        if (s->recheap_scratch >= s->recheap_size)
-            s->recheap_scratch = -1;
-    }
+    return buf;
 }
 
-#endif
 
+#ifdef GAGA
 // FIXME needs to be generalized. Should flexibly generate X lists per search
 static void extract_subject(struct session *s, const char *rec)
 {
@@ -564,76 +319,159 @@ static void extract_subject(struct session *s, const char *rec)
             assert(len < 1023);
             memcpy(buf, subfield, len);
             buf[len] = '\0';
+#ifdef FIXME
             if (*buf)
                 termlist_insert(s->termlist, buf);
+#endif
         }
     }
 }
+#endif
 
-static void pull_relevance_field(struct session *s, struct record *head, const char *rec,
-        char *field, int mult)
+static void add_facet(struct session *s, const char *type, const char *value)
 {
-    const char *fb;
-    while ((fb = find_field(rec, field)))
+    int i;
+
+    for (i = 0; i < s->num_termlists; i++)
+        if (!strcmp(s->termlists[i].name, type))
+            break;
+    if (i == s->num_termlists)
     {
-        char *ffield = strchr(fb, '\t');
-        if (!ffield)
-            return;
-        char *eol = strchr(ffield, '\n');
-        if (!eol)
-            return;
-        relevance_countwords(s->relevance, head, ffield, eol - ffield, mult);
-        rec = field + 1; // Crude way to cause a loop through repeating fields
+        if (i == SESSION_MAX_TERMLISTS)
+        {
+            yaz_log(YLOG_FATAL, "Too many termlists");
+            exit(1);
+        }
+        s->termlists[i].name = nmem_strdup(s->nmem, type);
+        s->termlists[i].termlist = termlist_create(s->nmem, s->expected_maxrecs, 15);
+        s->num_termlists = i + 1;
     }
+    termlist_insert(s->termlists[i].termlist, value);
 }
 
-static void pull_relevance_keys(struct session *s, struct record *head,  struct record *rec)
+static xmlDoc *normalize_record(struct client *cl, Z_External *rec)
 {
-    relevance_newrec(s->relevance, head);
-    pull_relevance_field(s, head, rec->buf, "100", 2);
-    pull_relevance_field(s, head, rec->buf, "245", 4);
-    //pull_relevance_field(s, head, rec->buf, "530", 1);
-    pull_relevance_field(s, head, rec->buf, "630", 1);
-    pull_relevance_field(s, head, rec->buf, "650", 1);
-    pull_relevance_field(s, head, rec->buf, "700", 1);
-    relevance_donerecord(s->relevance, head);
+    struct conf_retrievalprofile *rprofile = cl->database->rprofile;
+    struct conf_retrievalmap *m;
+    xmlNode *res;
+    xmlDoc *rdoc;
+
+    // First normalize to XML
+    if (rprofile->native_syntax == Nativesyn_iso2709)
+    {
+        char *buf;
+        int len;
+        if (rec->which != Z_External_octet)
+        {
+            yaz_log(YLOG_WARN, "Unexpected external branch, probably BER");
+            return 0;
+        }
+        buf = (char*) rec->u.octet_aligned->buf;
+        len = rec->u.octet_aligned->len;
+        if (yaz_marc_read_iso2709(rprofile->yaz_marc, buf, len) < 0)
+        {
+            yaz_log(YLOG_WARN, "Failed to decode MARC");
+            return 0;
+        }
+        if (yaz_marc_write_xml(rprofile->yaz_marc, &res,
+                    "http://www.loc.gov/MARC21/slim", 0, 0) < 0)
+        {
+            yaz_log(YLOG_WARN, "Failed to encode as XML");
+            return 0;
+        }
+        rdoc = xmlNewDoc("1.0");
+        xmlDocSetRootElement(rdoc, res);
+    }
+    else
+    {
+        yaz_log(YLOG_FATAL, "Unknown native_syntax in normalize_record");
+        exit(1);
+    }
+    for (m = rprofile->maplist; m; m = m->next)
+    {
+        xmlDoc *new;
+        if (m->type != Map_xslt)
+        {
+            yaz_log(YLOG_WARN, "Unknown map type");
+            return 0;
+        }
+        if (!(new = xsltApplyStylesheet(m->stylesheet, rdoc, 0)))
+        {
+            yaz_log(YLOG_WARN, "XSLT transformation failed");
+            return 0;
+        }
+        xmlFreeDoc(rdoc);
+        rdoc = new;
+    }
+    if (global_parameters.dump_records)
+    {
+        fprintf(stderr, "Record:\n----------------\n");
+        xmlDocFormatDump(stderr, rdoc, 1);
+    }
+    return rdoc;
 }
 
-static struct record *ingest_record(struct client *cl, char *buf, int len)
+static struct record *ingest_record(struct client *cl, Z_External *rec)
 {
+    xmlDoc *xdoc = normalize_record(cl, rec);
+    xmlNode *root, *n;
+    struct record *res, *head;
     struct session *se = cl->session;
-    struct record *res;
-    struct record *head;
-    const char *recbuf;
+    xmlChar *mergekey, *mergekey_norm;
 
-    wrbuf_rewind(se->wrbuf);
-    yaz_marc_xml(global_parameters.yaz_marc, YAZ_MARC_LINE);
-    if (yaz_marc_decode_wrbuf(global_parameters.yaz_marc, buf, len, se->wrbuf) < 0)
+    if (!xdoc)
+        return 0;
+
+    root = xmlDocGetRootElement(xdoc);
+    if (!(mergekey = xmlGetProp(root, "mergekey")))
     {
-        yaz_log(YLOG_WARN, "Failed to decode MARC record");
+        yaz_log(YLOG_WARN, "No mergekey found in record");
         return 0;
     }
-    wrbuf_putc(se->wrbuf, '\0');
-    recbuf = wrbuf_buf(se->wrbuf);
 
     res = nmem_malloc(se->nmem, sizeof(struct record));
-    res->buf = nmem_strdup(se->nmem, recbuf);
-
-    extract_subject(se, res->buf);
-
-    res->title = extract_title(se, res->buf);
-    res->merge_key = extract_mergekey(se, res->buf);
-    if (!res->merge_key)
-        return 0;
-    res->client = cl;
     res->next_cluster = 0;
     res->target_offset = -1;
     res->term_frequency_vec = 0;
+    res->title = "Unknown";
+    res->relevance = 0;
+
+    mergekey_norm = nmem_strdup(se->nmem, (char*) mergekey);
+    xmlFree(mergekey);
+    res->merge_key = normalize_mergekey(mergekey_norm);
 
     head = reclist_insert(se->reclist, res);
+    relevance_newrec(se->relevance, head);
 
-    pull_relevance_keys(se, head, res);
+    for (n = root->children; n; n = n->next)
+    {
+        if (n->type != XML_ELEMENT_NODE)
+            continue;
+        if (!strcmp(n->name, "facet"))
+        {
+            xmlChar *type = xmlGetProp(n, "type");
+            xmlChar *value = xmlNodeListGetString(xdoc, n->children, 0);
+            add_facet(se, type, value);
+            relevance_countwords(se->relevance, head, value, 1);
+            xmlFree(type);
+            xmlFree(value);
+        }
+        else if (!strcmp(n->name, "metadata"))
+        {
+            xmlChar *type = xmlGetProp(n, "type"), *value;
+            if (!strcmp(type, "title"))
+                res->title = nmem_strdup(se->nmem,
+                        value = xmlNodeListGetString(xdoc, n->children, 0));
+
+            relevance_countwords(se->relevance, head, value, 4);
+            xmlFree(type);
+            xmlFree(value);
+        }
+        else
+            yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name);
+    }
 
+    relevance_donerecord(se->relevance, head);
     se->total_records++;
 
     return res;
@@ -652,25 +490,14 @@ static void ingest_records(struct client *cl, Z_Records *r)
     for (i = 0; i < rlist->num_records; i++)
     {
         Z_NamePlusRecord *npr = rlist->records[i];
-        Z_External *e;
-        char *buf;
-        int len;
 
         if (npr->which != Z_NamePlusRecord_databaseRecord)
         {
             yaz_log(YLOG_WARN, "Unexpected record type, probably diagnostic");
             continue;
         }
-        e = npr->u.databaseRecord;
-        if (e->which != Z_External_octet)
-        {
-            yaz_log(YLOG_WARN, "Unexpected external branch, probably BER");
-            continue;
-        }
-        buf = (char*) e->u.octet_aligned->buf;
-        len = e->u.octet_aligned->len;
 
-        rec = ingest_record(cl, buf, len);
+        rec = ingest_record(cl, npr->u.databaseRecord);
         if (!rec)
             continue;
     }
@@ -678,17 +505,6 @@ static void ingest_records(struct client *cl, Z_Records *r)
         session_alert_watch(s, SESSION_WATCH_RECORDS);
 }
 
-xsltStylesheetPtr load_stylesheet(const char *fname)
-{
-    xsltStylesheetPtr ret;
-    if (!(ret = xsltParseStylesheetFile((const xmlChar *) fname)))
-    {
-        yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to load stylesheet %s", fname);
-        exit(1);
-    }
-    return ret;
-}
-
 static void do_presentResponse(IOCHAN i, Z_APDU *a)
 {
     struct connection *co = iochan_getdata(i);
@@ -975,6 +791,8 @@ static int client_prep_connection(struct client *cl)
         return 0;
 }
 
+// This function will most likely vanish when a proper target profile mechanism is
+// introduced.
 void load_simpletargets(const char *fn)
 {
     FILE *f = fopen(fn, "r");
@@ -1059,6 +877,8 @@ void load_simpletargets(const char *fn)
         database->databases[0] = xstrdup(db);
         database->databases[1] = 0;
         database->errors = 0;
+        database->qprofile = 0;
+        database->rprofile = database_retrieval_profile(database);
         database->next = databases;
         databases = database;
 
@@ -1158,6 +978,21 @@ void session_alert_watch(struct session *s, int what)
     s->watchlist[what].data = 0;
 }
 
+// This needs to be extended with selection criteria
+static struct conf_retrievalprofile *database_retrieval_profile(struct database *db)
+{
+    if (!config)
+    {
+        yaz_log(YLOG_FATAL, "Must load configuration (-f)");
+        exit(1);
+    }
+    if (!config->retrievalprofiles)
+    {
+        yaz_log(YLOG_FATAL, "No retrieval profiles defined");
+    }
+    return config->retrievalprofiles;
+}
+
 // This should be extended with parameters to control selection criteria
 // Associates a set of clients with a session;
 int select_targets(struct session *se)
@@ -1202,11 +1037,12 @@ char *search(struct session *se, char *query)
     {
         char *p[512];
         int maxrecs = live_channels * global_parameters.toget;
-        se->termlist = termlist_create(se->nmem, maxrecs, 15);
+        se->num_termlists = 0;
         se->reclist = reclist_create(se->nmem, maxrecs);
         extract_terms(se->nmem, query, p);
         se->relevance = relevance_create(se->nmem, (const char **) p, maxrecs);
         se->total_records = se->total_hits = 0;
+        se->expected_maxrecs = maxrecs;
     }
     else
         return "NOTARGETS";
@@ -1232,10 +1068,11 @@ struct session *new_session()
     
     session->total_hits = 0;
     session->total_records = 0;
-    session->termlist = 0;
+    session->num_termlists = 0;
     session->reclist = 0;
     session->requestid = -1;
     session->clients = 0;
+    session->expected_maxrecs = 0;
     session->query[0] = '\0';
     session->nmem = nmem_create();
     session->wrbuf = wrbuf_alloc();
@@ -1270,9 +1107,14 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count)
     return res;
 }
 
-struct termlist_score **termlist(struct session *s, int *num)
+struct termlist_score **termlist(struct session *s, const char *name, int *num)
 {
-    return termlist_highscore(s->termlist, num);
+    int i;
+
+    for (i = 0; i < s->num_termlists; i++)
+        if (!strcmp(s->termlists[i].name, name))
+            return termlist_highscore(s->termlists[i].termlist, num);
+    return 0;
 }
 
 #ifdef REPORT_NMEM
@@ -1373,7 +1215,7 @@ int main(int argc, char **argv)
 
     yaz_log_init(YLOG_DEFAULT_LEVEL, "pazpar2", 0);
 
-    while ((ret = options("f:x:c:h:p:C:s:", argv, argc, &arg)) != -2)
+    while ((ret = options("f:x:c:h:p:C:s:d", argv, argc, &arg)) != -2)
     {
 	switch (ret) {
             case 'f':
@@ -1397,8 +1239,8 @@ int main(int argc, char **argv)
             case 's':
                 load_simpletargets(arg);
                 break;
-            case 'x':
-                global_parameters.xsl = load_stylesheet(arg);
+            case 'd':
+                global_parameters.dump_records = 1;
                 break;
 	    default:
 		fprintf(stderr, "Usage: pazpar2\n"
@@ -1418,8 +1260,6 @@ int main(int argc, char **argv)
         exit(1);
     }
 
-    if (!global_parameters.xsl)
-        global_parameters.xsl = load_stylesheet("../etc/default.xsl");
     global_parameters.ccl_filter = load_cclfile("../etc/default.bib");
     global_parameters.yaz_marc = yaz_marc_create();
     yaz_marc_subfield_str(global_parameters.yaz_marc, "\t");
diff --git a/src/pazpar2.h b/src/pazpar2.h
index b2d3d95..7677d9d 100644
--- a/src/pazpar2.h
+++ b/src/pazpar2.h
@@ -16,11 +16,13 @@ struct record;
 #include "termlists.h"
 #include "relevance.h"
 #include "eventl.h"
+#include "config.h"
+
+struct client;
 
 struct record {
     struct client *client;
     int target_offset;
-    char *buf;
     char *merge_key;
     char *title;
     int relevance;
@@ -44,10 +46,11 @@ struct database {
     char *url;
     char **databases;
     int errors;
+    struct conf_queryprofile *qprofile;
+    struct conf_retrievalprofile *rprofile;
     struct database *next;
 };
 
-struct client;
 
 // Represents a physical, reusable  connection to a remote Z39.50 host
 struct connection {
@@ -94,8 +97,16 @@ struct client {
 #define SESSION_WATCH_RECORDS   0
 #define SESSION_WATCH_MAX       0
 
+#define SESSION_MAX_TERMLISTS 10
+
 typedef void (*session_watchfun)(void *data);
 
+struct named_termlist
+{
+    char *name;
+    struct termlist *termlist;
+};
+
 // End-user session
 struct session {
     struct client *clients;
@@ -103,13 +114,15 @@ struct session {
     char query[1024];
     NMEM nmem;          // Nmem for each operation (i.e. search)
     WRBUF wrbuf;        // Wrbuf for scratch(i.e. search)
-    struct termlist *termlist;
+    int num_termlists;
+    struct named_termlist termlists[SESSION_MAX_TERMLISTS];
     struct relevance *relevance;
     struct reclist *reclist;
     struct {
         void *data;
         session_watchfun fun;
     } watchlist[SESSION_WATCH_MAX + 1];
+    int expected_maxrecs;
     int total_hits;
     int total_records;
 };
@@ -138,6 +151,7 @@ struct hitsbytarget {
 };
 
 struct parameters {
+    int dump_records;
     int timeout;		/* operations timeout, in seconds */
     char implementationId[128];
     char implementationName[128];
@@ -150,7 +164,6 @@ struct parameters {
     yaz_marc_t yaz_marc;
     ODR odr_out;
     ODR odr_in;
-    xsltStylesheetPtr xsl;
 };
 
 struct hitsbytarget *hitsbytarget(struct session *s, int *count);
@@ -161,7 +174,7 @@ int load_targets(struct session *s, const char *fn);
 void statistics(struct session *s, struct statistics *stat);
 char *search(struct session *s, char *query);
 struct record **show(struct session *s, int start, int *num, int *total, int *sumhits, NMEM nmem_show);
-struct termlist_score **termlist(struct session *s, int *num);
+struct termlist_score **termlist(struct session *s, const char *name, int *num);
 void session_set_watch(struct session *s, int what, session_watchfun fun, void *data);
 
 #endif
diff --git a/src/relevance.c b/src/relevance.c
index 4597c67..d689e7b 100644
--- a/src/relevance.c
+++ b/src/relevance.c
@@ -1,5 +1,5 @@
 /*
- * $Id: relevance.c,v 1.2 2006-12-20 22:18:33 adam Exp $
+ * $Id: relevance.c,v 1.3 2007-01-03 06:23:44 quinn Exp $
  */
 
 #include <ctype.h>
@@ -67,16 +67,16 @@ static void word_trie_addterm(NMEM nmem, struct word_trie *n, const char *term,
 
 #define raw_char(c) (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' : -1)
 
-static int word_trie_match(struct word_trie *t, const char *word, int len, int *skipped)
+static int word_trie_match(struct word_trie *t, const char *word, int *skipped)
 {
     int c = raw_char(tolower(*word));
 
-    if (!len)
+    if (!*word)
         return 0;
 
-    word++; len--;
+    word++;
     (*skipped)++;
-    if (!len || raw_char(*word) < 0)
+    if (!*word || raw_char(*word) < 0)
     {
         if (t->list[c].termno > 0)
             return t->list[c].termno;
@@ -87,7 +87,7 @@ static int word_trie_match(struct word_trie *t, const char *word, int len, int *
     {
         if (t->list[c].child)
         {
-            return word_trie_match(t->list[c].child, word, len, skipped);
+            return word_trie_match(t->list[c].child, word, skipped);
         }
         else
             return 0;
@@ -136,34 +136,27 @@ void relevance_newrec(struct relevance *r, struct record *rec)
 // FIXME. The definition of a word is crude here.. should support
 // some form of localization mechanism?
 void relevance_countwords(struct relevance *r, struct record *head,
-        const char *words, int len, int multiplier)
+        const char *words, int multiplier)
 {
-    while (len)
+    while (*words)
     {
         char c;
         int res;
         int skipped;
-        while (len && (c = raw_char(tolower(*words))) < 0)
-        {
+        while (*words && (c = raw_char(tolower(*words))) < 0)
             words++;
-            len--;
-        }
-        if (!len)
+        if (!*words)
             return;
         skipped = 0;
-        if ((res = word_trie_match(r->wt, words, len, &skipped)))
+        if ((res = word_trie_match(r->wt, words, &skipped)))
         {
             words += skipped;
-            len -= skipped;
             head->term_frequency_vec[res] += multiplier;
         }
         else
         {
-            while (len && (c = raw_char(tolower(*words))) >= 0)
-            {
+            while (*words && (c = raw_char(tolower(*words))) >= 0)
                 words++;
-                len--;
-            }
         }
         head->term_frequency_vec[0]++;
     }
diff --git a/src/relevance.h b/src/relevance.h
index 38c3d9c..1639cf3 100644
--- a/src/relevance.h
+++ b/src/relevance.h
@@ -11,7 +11,7 @@ struct relevance;
 struct relevance *relevance_create(NMEM nmem, const char **terms, int numrecs);
 void relevance_newrec(struct relevance *r, struct record *rec);
 void relevance_countwords(struct relevance *r, struct record *rec,
-        const char *words, int len, int multiplier);
+        const char *words, int multiplier);
 void relevance_donerecord(struct relevance *r, struct record *rec);
 
 void relevance_prepare_read(struct relevance *rel, struct reclist *rec);