1 /* $Id: rpnscan.c,v 1.3 2006-09-21 20:22:34 adam Exp $
2 Copyright (C) 1995-2006
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
33 #include <yaz/diagbib1.h>
35 #include <zebra_xpath.h>
36 #include <yaz/wrbuf.h>
41 #define RPN_MAX_ORDS 32
43 int log_scan = YLOG_LOG;
45 /* convert APT SCAN term to internal cmap */
46 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
47 char *termz, int reg_type)
49 char termz0[IT_MAX_WORD];
51 if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
52 return ZEBRA_FAIL; /* error */
56 const char *cp = (const char *) termz0;
57 const char *cp_end = cp + strlen(cp);
60 const char *space_map = NULL;
63 while ((len = (cp_end - cp)) > 0)
65 map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
66 if (**map == *CHR_SPACE)
71 for (src = space_map; *src; src++)
74 for (src = *map; *src; src++)
83 static void count_set(ZebraHandle zh, RSET rset, zint *count)
89 yaz_log(YLOG_DEBUG, "count_set");
91 rset->hits_limit = zh->approx_limit;
94 rfd = rset_open(rset, RSETF_READ);
95 while (rset_read(rfd, &key,0 /* never mind terms */))
97 if (key.mem[0] != psysno)
100 if (rfd->counted_items >= rset->hits_limit)
105 *count = rset->hits_count;
108 struct scan2_info_entry {
115 static int scan_handle2(char *name, const char *info, int pos, void *client)
118 struct scan2_info_entry *scan_info = (struct scan2_info_entry *) client;
120 if (scan_info->pos_to_save != pos)
123 len_prefix = strlen(scan_info->prefix);
124 if (memcmp (name, scan_info->prefix, len_prefix))
127 /* skip special terms such as first-in-field specials */
128 if (name[len_prefix] < CHR_BASE_CHAR)
131 wrbuf_rewind(scan_info->term);
132 wrbuf_puts(scan_info->term, name+len_prefix);
134 assert (*info == sizeof(ISAM_P));
135 memcpy (&scan_info->isam_p, info+1, sizeof(ISAM_P));
140 static int scan_save_set(ZebraHandle zh, ODR stream, NMEM nmem,
141 struct rset_key_control *kc,
142 Z_AttributesPlusTerm *zapt,
146 struct scan2_info_entry *ar, int ord_no,
147 ZebraScanEntry *glist, int pos)
151 for (i = 0; i < ord_no; i++)
153 if (ar[i].isam_p && strcmp(wrbuf_buf(ar[i].term), term) == 0)
155 RSET rset_t = rset_trunc(
156 zh, &ar[i].isam_p, 1,
157 wrbuf_buf(ar[i].term), wrbuf_len(ar[i].term),
158 NULL, 0, zapt->term->which, nmem,
159 kc, kc->scope, 0, index_type,
161 0 /* term_ref_id_str */);
170 rset = rset_create_or(nmem, kc, kc->scope, 0 /* termid */,
179 /* merge with limit_set if given */
184 rsets[1] = rset_dup(limit_set);
186 rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
189 count_set(zh, rset, &count);
195 zebra_term_untrans_iconv(zh, stream->mem, index_type,
196 &glist[pos].term, term);
197 glist[pos].occurrences = count;
205 static ZEBRA_RES rpn_scan_ver2(ZebraHandle zh, ODR stream, NMEM nmem,
206 struct rset_key_control *kc,
207 Z_AttributesPlusTerm *zapt,
208 int *position, int *num_entries,
209 ZebraScanEntry **list,
210 int *is_partial, RSET limit_set,
211 int index_type, int ord_no, int *ords)
213 struct scan2_info_entry *ar = nmem_malloc(nmem, sizeof(*ar) * ord_no);
214 struct rpn_char_map_info rcmi;
218 ZebraScanEntry *glist = (ZebraScanEntry *)
219 odr_malloc(stream, *num_entries * sizeof(*glist));
222 if (*position > *num_entries+1)
229 rpn_char_map_prepare (zh->reg, index_type, &rcmi);
231 for (i = 0; i < ord_no; i++)
232 ar[i].term = wrbuf_alloc();
234 for (i = 0; i < ord_no; i++)
236 char termz[IT_MAX_WORD+20];
239 prefix_len = key_SU_encode (ords[i], termz);
240 termz[prefix_len] = 0;
241 strcpy(ar[i].prefix, termz);
243 if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
246 wrbuf_rewind(ar[i].term);
247 wrbuf_puts(ar[i].term, termz + prefix_len);
250 /** deal with terms before position .. */
251 /* the glist index starts at zero (unlike scan positions */
252 for (pos = *position-2; pos >= 0; )
256 /* scan on all maximum terms */
257 for (i = 0; i < ord_no; i++)
259 if (ar[i].isam_p == 0)
261 char termz[IT_MAX_WORD+20];
265 ar[i].pos_to_save = -1;
267 strcpy(termz, ar[i].prefix);
268 strcat(termz, wrbuf_buf(ar[i].term));
269 dict_scan(zh->reg->dict, termz, &before, &after,
273 /* get maximum after scan */
274 for (i = 0; i < ord_no; i++)
277 && (hi == 0 || strcmp(wrbuf_buf(ar[i].term), hi) > 0))
278 hi = wrbuf_buf(ar[i].term);
282 if (scan_save_set(zh, stream, nmem, kc, zapt, limit_set, hi,
283 index_type, ar, ord_no, glist,
284 (pos >= 0 && pos < *num_entries) ? pos : -1))
287 /* see if we got all terms before.. */
291 /* did not get all terms; adjust the real position and reduce
293 yaz_log(YLOG_LOG, "before terms dif=%d", dif);
299 for (i = 0; i < ord_no; i++)
301 char termz[IT_MAX_WORD+20];
304 prefix_len = key_SU_encode (ords[i], termz);
305 termz[prefix_len] = 0;
306 strcpy(ar[i].prefix, termz);
308 if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
311 wrbuf_rewind(ar[i].term);
312 wrbuf_puts(ar[i].term, termz + prefix_len);
316 for (pos = *position-1; pos < *num_entries; )
320 /* scan on all minimum terms */
321 for (i = 0; i < ord_no; i++)
323 if (ar[i].isam_p == 0)
325 char termz[IT_MAX_WORD+20];
327 int after = (pos == *position-1) ? 1 : 2;
329 ar[i].pos_to_save = 1;
331 strcpy(termz, ar[i].prefix);
332 strcat(termz, wrbuf_buf(ar[i].term));
333 dict_scan(zh->reg->dict, termz, &before, &after,
337 /* get minimum after scan */
338 for (i = 0; i < ord_no; i++)
341 && (lo == 0 || strcmp(wrbuf_buf(ar[i].term), lo) < 0))
342 lo = wrbuf_buf(ar[i].term);
346 if (scan_save_set(zh, stream, nmem, kc, zapt, limit_set, lo,
347 index_type, ar, ord_no, glist,
348 (pos >= 0 && pos < *num_entries) ? pos : -1))
352 if (pos != *num_entries)
366 struct scan1_info_entry {
372 struct scan1_info_entry *list;
378 static int scan_handle1(char *name, const char *info, int pos, void *client)
381 struct scan_info *scan_info = (struct scan_info *) client;
383 len_prefix = strlen(scan_info->prefix);
384 if (memcmp (name, scan_info->prefix, len_prefix))
387 idx = scan_info->after - pos + scan_info->before;
391 /* skip special terms such as first-in-field specials */
392 if (name[len_prefix] < CHR_BASE_CHAR)
397 scan_info->list[idx].term = (char *)
398 odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
399 strcpy(scan_info->list[idx].term, name + len_prefix);
400 assert (*info == sizeof(ISAM_P));
401 memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
405 static ZEBRA_RES rpn_scan_ver1(ZebraHandle zh, ODR stream, NMEM rset_nmem,
406 struct rset_key_control *kc,
407 Z_AttributesPlusTerm *zapt,
408 int *position, int *num_entries,
409 ZebraScanEntry **list,
410 int *is_partial, RSET limit_set,
411 int index_type, int ord_no, int *ords)
414 int num = *num_entries;
418 struct scan_info *scan_info_array;
419 char termz[IT_MAX_WORD+20];
420 ZebraScanEntry *glist;
421 int ptr[RPN_MAX_ORDS];
429 yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
430 "after=%d before+after=%d",
431 pos, num, before, after, before+after);
432 scan_info_array = (struct scan_info *)
433 odr_malloc(stream, ord_no * sizeof(*scan_info_array));
434 for (i = 0; i < ord_no; i++)
436 int j, prefix_len = 0;
437 int before_tmp = before, after_tmp = after;
438 struct scan_info *scan_info = scan_info_array + i;
439 struct rpn_char_map_info rcmi;
441 rpn_char_map_prepare (zh->reg, index_type, &rcmi);
443 scan_info->before = before;
444 scan_info->after = after;
445 scan_info->odr = stream;
447 scan_info->list = (struct scan1_info_entry *)
448 odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
449 for (j = 0; j<before+after; j++)
450 scan_info->list[j].term = NULL;
452 prefix_len += key_SU_encode (ords[i], termz + prefix_len);
453 termz[prefix_len] = 0;
454 strcpy(scan_info->prefix, termz);
456 if (trans_scan_term(zh, zapt, termz+prefix_len, index_type) ==
460 dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
461 scan_info, scan_handle1);
463 glist = (ZebraScanEntry *)
464 odr_malloc(stream, (before+after)*sizeof(*glist));
466 /* consider terms after main term */
467 for (i = 0; i < ord_no; i++)
471 for (i = 0; i<after; i++)
474 const char *mterm = NULL;
477 int lo = i + pos-1; /* offset in result list */
479 /* find: j0 is the first of the minimal values */
480 for (j = 0; j < ord_no; j++)
482 if (ptr[j] < before+after && ptr[j] >= 0 &&
483 (tst = scan_info_array[j].list[ptr[j]].term) &&
484 (!mterm || strcmp (tst, mterm) < 0))
491 break; /* no value found, stop */
493 /* get result set for first one , but only if it's within bounds */
496 /* get result set for first term */
497 zebra_term_untrans_iconv(zh, stream->mem, index_type,
498 &glist[lo].term, mterm);
499 rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
500 glist[lo].term, strlen(glist[lo].term),
501 NULL, 0, zapt->term->which, rset_nmem,
502 kc, kc->scope, 0, index_type, 0 /* hits_limit */,
503 0 /* term_ref_id_str */);
505 ptr[j0]++; /* move index for this set .. */
506 /* get result set for remaining scan terms */
507 for (j = j0+1; j<ord_no; j++)
509 if (ptr[j] < before+after && ptr[j] >= 0 &&
510 (tst = scan_info_array[j].list[ptr[j]].term) &&
511 !strcmp (tst, mterm))
520 zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
522 strlen(glist[lo].term), NULL, 0,
523 zapt->term->which,rset_nmem,
524 kc, kc->scope, 0, index_type, 0 /* hits_limit */,
525 0 /* term_ref_id_str */ );
526 rset = rset_create_or(rset_nmem, kc,
527 kc->scope, 0 /* termid */,
536 /* merge with limit_set if given */
541 rsets[1] = rset_dup(limit_set);
543 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
546 count_set(zh, rset, &count);
547 glist[lo].occurrences = count;
553 *num_entries -= (after-i);
555 if (*num_entries < 0)
561 /* consider terms before main term */
562 for (i = 0; i<ord_no; i++)
565 for (i = 0; i<before; i++)
568 const char *mterm = NULL;
571 int lo = before-1-i; /* offset in result list */
574 for (j = 0; j <ord_no; j++)
576 if (ptr[j] < before && ptr[j] >= 0 &&
577 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
578 (!mterm || strcmp (tst, mterm) > 0))
587 zebra_term_untrans_iconv(zh, stream->mem, index_type,
588 &glist[lo].term, mterm);
591 (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
592 glist[lo].term, strlen(glist[lo].term),
593 NULL, 0, zapt->term->which, rset_nmem,
594 kc, kc->scope, 0, index_type, 0 /* hits_limit */,
595 0 /* term_ref_id_str */);
599 for (j = j0+1; j<ord_no; j++)
601 if (ptr[j] < before && ptr[j] >= 0 &&
602 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
603 !strcmp (tst, mterm))
608 rsets[1] = rset_trunc(
610 &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
612 strlen(glist[lo].term), NULL, 0,
613 zapt->term->which, rset_nmem,
614 kc, kc->scope, 0, index_type, 0 /* hits_limit */,
615 0 /* term_ref_id_str */);
616 rset = rset_create_or(rset_nmem, kc,
617 kc->scope, 0 /* termid */, 2, rsets);
626 rsets[1] = rset_dup(limit_set);
628 rset = rset_create_and(rset_nmem, kc, kc->scope, 2, rsets);
630 count_set(zh, rset, &count);
631 glist[lo].occurrences = count;
640 if (*num_entries <= 0)
647 *list = glist + i; /* list is set to first 'real' entry */
649 yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
650 *position, *num_entries);
655 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
656 oid_value attributeset,
657 int num_bases, char **basenames,
658 int *position, int *num_entries, ZebraScanEntry **list,
659 int *is_partial, RSET limit_set)
662 int ords[RPN_MAX_ORDS], ord_no = 0;
665 char *search_type = NULL;
671 struct rset_key_control *kc = 0;
676 if (attributeset == VAL_NONE)
677 attributeset = VAL_BIB1;
679 if (!limit_set) /* no limit set given already */
681 /* see if there is a @attr 8=set */
683 int termset_value_numeric;
684 const char *termset_value_string;
685 attr_init_APT(&termset, zapt, 8);
686 termset_value_numeric =
687 attr_find_ex(&termset, NULL, &termset_value_string);
688 if (termset_value_numeric != -1)
691 const char *termset_name = 0;
693 if (termset_value_numeric != -2)
696 sprintf(resname, "%d", termset_value_numeric);
697 termset_name = resname;
700 termset_name = termset_value_string;
702 limit_set = resultSetRef (zh, termset_name);
706 yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
707 *position, *num_entries, attributeset);
709 if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &index_type, &search_type,
710 rank_type, &complete_flag, &sort_flag))
713 zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
716 if (num_bases > RPN_MAX_ORDS)
718 zebra_setError(zh, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
722 for (base_no = 0; base_no < num_bases; base_no++)
726 if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
728 zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
733 if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeset, &ord)
736 ords[ord_no++] = ord;
740 *num_entries = 0; /* zebra_apt_get_ord should set error reason */
743 if (*num_entries < 1)
748 nmem = nmem_create();
749 kc = zebra_key_control_create(zh);
751 res = rpn_scan_ver2(zh, stream, nmem, kc, zapt, position, num_entries,
753 is_partial, limit_set, index_type, ord_no, ords);
762 * indent-tabs-mode: nil
764 * vim: shiftwidth=4 tabstop=8 expandtab