2 * Copyright (C) 1994-1997, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.63 1997-09-17 12:19:18 adam
8 * Zebra version corresponds to YAZ version 1.4.
9 * Changed Zebra server so that it doesn't depend on global common_resource.
11 * Revision 1.62 1997/09/05 15:30:09 adam
12 * Changed prototype for chr_map_input - added const.
13 * Added support for C++, headers uses extern "C" for public definitions.
15 * Revision 1.61 1997/02/10 10:21:14 adam
16 * Bug fix: in search terms character (^) wasn't observed.
18 * Revision 1.60 1997/01/31 11:10:34 adam
19 * Bug fix: Leading and trailing white space weren't removed in scan tokens.
21 * Revision 1.59 1997/01/17 11:31:46 adam
22 * Bug fix: complete phrase search didn't work.
24 * Revision 1.58 1996/12/23 15:30:45 adam
26 * Bug fix: result sets weren't deleted after server shut down.
28 * Revision 1.57 1996/11/11 13:38:02 adam
29 * Added proximity support in search.
31 * Revision 1.56 1996/11/08 11:10:32 adam
32 * Buffers used during file match got bigger.
33 * Compressed ISAM support everywhere.
34 * Bug fixes regarding masking characters in queries.
35 * Redesigned Regexp-2 queries.
37 * Revision 1.55 1996/11/04 14:07:44 adam
38 * Moved truncation code to trunc.c.
40 * Revision 1.54 1996/10/29 14:09:52 adam
41 * Use of cisam system - enabled if setting isamc is 1.
43 * Revision 1.53 1996/06/26 09:21:43 adam
44 * Bug fix: local attribute set wasn't obeyed in scan.
46 * Revision 1.52 1996/06/17 14:26:20 adam
47 * Function gen_regular_rel changed to handle negative numbers.
49 * Revision 1.51 1996/06/11 10:54:15 quinn
52 * Revision 1.50 1996/06/07 08:51:53 adam
53 * Bug fix: Character mapping was broken (introducued by last revision).
55 * Revision 1.49 1996/06/04 10:18:11 adam
56 * Search/scan uses character mapping module.
58 * Revision 1.48 1996/05/28 15:15:01 adam
59 * Bug fix: Didn't handle unknown database correctly.
61 * Revision 1.47 1996/05/15 18:36:28 adam
62 * Function trans_term transforms unsearchable characters to blanks.
64 * Revision 1.46 1996/05/15 11:57:56 adam
65 * Fixed bug introduced by set/field mapping in search operations.
67 * Revision 1.45 1996/05/14 11:34:00 adam
68 * Scan support in multiple registers/databases.
70 * Revision 1.44 1996/05/14 06:16:44 adam
71 * Compact use/set bytes used in search service.
73 * Revision 1.43 1996/05/09 09:54:43 adam
74 * Server supports maps from one logical attributes to a list of physical
76 * The extraction process doesn't make space consuming 'any' keys.
78 * Revision 1.42 1996/05/09 07:28:56 quinn
79 * Work towards phrases and multiple registers
81 * Revision 1.41 1996/03/20 09:36:43 adam
82 * Function dict_lookup_grep got extra parameter, init_pos, which marks
83 * from which position in pattern approximate pattern matching should occur.
84 * Approximate pattern matching is used in relevance=re-2.
86 * Revision 1.40 1996/02/02 13:44:44 adam
87 * The public dictionary functions simply use char instead of Dict_char
88 * to represent search strings. Dict_char is used internally only.
90 * Revision 1.39 1996/01/03 16:22:13 quinn
93 * Revision 1.38 1995/12/11 09:12:55 adam
94 * The rec_get function returns NULL if record doesn't exist - will
95 * happen in the server if the result set records have been deleted since
96 * the creation of the set (i.e. the search).
97 * The server saves a result temporarily if it is 'volatile', i.e. the
98 * set is register dependent.
100 * Revision 1.37 1995/12/06 15:05:28 adam
101 * More verbose in count_set.
103 * Revision 1.36 1995/12/06 12:41:27 adam
104 * New command 'stat' for the index program.
105 * Filenames can be read from stdin by specifying '-'.
106 * Bug fix/enhancement of the transformation from terms to regular
107 * expressons in the search engine.
109 * Revision 1.35 1995/11/27 09:29:00 adam
110 * Bug fixes regarding conversion to regular expressions.
112 * Revision 1.34 1995/11/16 17:00:56 adam
113 * Better logging of rpn query.
115 * Revision 1.33 1995/11/01 13:58:28 quinn
116 * Moving data1 to yaz/retrieval
118 * Revision 1.32 1995/10/27 14:00:11 adam
119 * Implemented detection of database availability.
121 * Revision 1.31 1995/10/17 18:02:10 adam
122 * New feature: databases. Implemented as prefix to words in dictionary.
124 * Revision 1.30 1995/10/16 09:32:38 adam
125 * More work on relational op.
127 * Revision 1.29 1995/10/13 16:01:49 adam
130 * Revision 1.28 1995/10/13 12:26:43 adam
131 * Optimization of truncation.
133 * Revision 1.27 1995/10/12 17:07:22 adam
136 * Revision 1.26 1995/10/12 12:40:54 adam
137 * Bug fixes in rpn_prox.
139 * Revision 1.25 1995/10/10 13:59:24 adam
140 * Function rset_open changed its wflag parameter to general flags.
142 * Revision 1.24 1995/10/09 16:18:37 adam
143 * Function dict_lookup_grep got extra client data parameter.
145 * Revision 1.23 1995/10/06 16:33:37 adam
146 * Use attribute mappings.
148 * Revision 1.22 1995/10/06 15:07:39 adam
149 * Structure 'local-number' handled.
151 * Revision 1.21 1995/10/06 13:52:06 adam
152 * Bug fixes. Handler may abort further scanning.
154 * Revision 1.20 1995/10/06 11:06:33 adam
155 * Scan entries include 'occurrences' now.
157 * Revision 1.19 1995/10/06 10:43:56 adam
158 * Scan added. 'occurrences' in scan entries not set yet.
160 * Revision 1.18 1995/10/04 16:57:20 adam
161 * Key input and merge sort in one pass.
163 * Revision 1.17 1995/10/04 12:55:17 adam
164 * Bug fix in ranked search. Use=Any keys inserted.
166 * Revision 1.16 1995/10/02 16:24:40 adam
167 * Use attribute actually used in search requests.
169 * Revision 1.15 1995/10/02 15:18:52 adam
170 * New member in recRetrieveCtrl: diagnostic.
172 * Revision 1.14 1995/09/28 12:10:32 adam
173 * Bug fixes. Field prefix used in queries.
175 * Revision 1.13 1995/09/18 14:17:50 adam
178 * Revision 1.12 1995/09/15 14:45:21 adam
180 * Work on truncation.
182 * Revision 1.11 1995/09/14 11:53:27 adam
183 * First work on regular expressions/truncations.
185 * Revision 1.10 1995/09/11 15:23:26 adam
186 * More work on relevance search.
188 * Revision 1.9 1995/09/11 13:09:35 adam
189 * More work on relevance feedback.
191 * Revision 1.8 1995/09/08 14:52:27 adam
192 * Minor changes. Dictionary is lower case now.
194 * Revision 1.7 1995/09/07 13:58:36 adam
195 * New parameter: result-set file descriptor (RSFD) to support multiple
196 * positions within the same result-set.
197 * Boolean operators: and, or, not implemented.
198 * Result-set references.
200 * Revision 1.6 1995/09/06 16:11:18 adam
201 * Option: only one word key per file.
203 * Revision 1.5 1995/09/06 10:33:04 adam
204 * More work on present. Some log messages removed.
206 * Revision 1.4 1995/09/05 15:28:40 adam
207 * More work on search engine.
209 * Revision 1.3 1995/09/04 15:20:22 adam
212 * Revision 1.2 1995/09/04 12:33:43 adam
213 * Various cleanup. YAZ util used instead.
215 * Revision 1.1 1995/09/04 09:10:40 adam
216 * More work on index add/del/update.
217 * Merge sort implemented.
218 * Initial work on z39 server.
238 Z_AttributesPlusTerm *zapt;
241 static int attr_find (AttrType *src, oid_value *attributeSetP)
243 while (src->major < src->zapt->num_attributes)
245 Z_AttributeElement *element;
247 element = src->zapt->attributeList[src->major];
248 if (src->type == *element->attributeType)
250 switch (element->which)
252 case Z_AttributeValue_numeric:
254 if (element->attributeSet && attributeSetP)
258 attrset = oid_getentbyoid (element->attributeSet);
259 *attributeSetP = attrset->value;
261 return *element->value.numeric;
263 case Z_AttributeValue_complex:
264 if (src->minor >= element->value.complex->num_list ||
265 element->value.complex->list[src->minor]->which !=
266 Z_StringOrNumeric_numeric)
269 if (element->attributeSet && attributeSetP)
273 attrset = oid_getentbyoid (element->attributeSet);
274 *attributeSetP = attrset->value;
276 return *element->value.complex->list[src->minor-1]->u.numeric;
286 static void attr_init (AttrType *src, Z_AttributesPlusTerm *zapt,
306 static void add_isam_p (const char *info, struct grep_info *p)
308 if (p->isam_p_indx == p->isam_p_size)
310 ISAM_P *new_isam_p_buf;
315 p->isam_p_size = 2*p->isam_p_size + 100;
316 new_isam_p_buf = xmalloc (sizeof(*new_isam_p_buf) *
320 memcpy (new_isam_p_buf, p->isam_p_buf,
321 p->isam_p_indx * sizeof(*p->isam_p_buf));
322 xfree (p->isam_p_buf);
324 p->isam_p_buf = new_isam_p_buf;
327 new_term_no = xmalloc (sizeof(*new_term_no) *
331 memcpy (new_term_no, p->isam_p_buf,
332 p->isam_p_indx * sizeof(*p->term_no));
335 p->term_no = new_term_no;
338 assert (*info == sizeof(*p->isam_p_buf));
339 memcpy (p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
343 static int grep_handle (char *name, const char *info, void *p)
345 add_isam_p (info, p);
349 static int term_pre (const char **src, const char *ct1, const char *ct2)
351 const char *s1, *s0 = *src;
354 /* skip white space */
357 if (ct1 && strchr (ct1, *s0))
359 if (ct2 && strchr (ct2, *s0))
362 map = map_chrs_input (&s1, strlen(s1));
363 if (**map != *CHR_SPACE)
371 static int term_100 (const char **src, char *dst, int space_split)
377 if (!term_pre (src, NULL, NULL))
383 map = map_chrs_input (&s0, strlen(s0));
384 if (space_split && **map == *CHR_SPACE)
398 static int term_101 (const char **src, char *dst, int space_split)
404 if (!term_pre (src, "#", "#"))
418 map = map_chrs_input (&s0, strlen(s0));
419 if (space_split && **map == *CHR_SPACE)
435 static int term_103 (const char **src, char *dst, int *errors, int space_split)
441 if (!term_pre (src, "^\\()[].*+?|", "("))
444 if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
447 *errors = s0[1] - '0';
454 if (strchr ("^\\()[].*+?|-", *s0))
459 map = map_chrs_input (&s0, strlen(s0));
460 if (**map == *CHR_SPACE)
475 static int term_102 (const char **src, char *dst, int space_split)
477 return term_103 (src, dst, NULL, space_split);
480 /* gen_regular_rel - generate regular expression from relation
481 * val: border value (inclusive)
482 * islt: 1 if <=; 0 if >=.
484 static void gen_regular_rel (char *dst, int val, int islt)
491 logf (LOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
495 strcpy (dst, "(-[0-9]+|");
503 strcpy (dst, "([0-9]+|-");
504 dst_p = strlen (dst);
514 dst_p = strlen (dst);
515 sprintf (numstr, "%d", val);
516 for (w = strlen(numstr); --w >= 0; pos++)
535 strcpy (dst + dst_p, numstr);
536 dst_p = strlen(dst) - pos - 1;
564 for (i = 0; i<pos; i++)
577 for (i=1; i<pos; i++)
578 strcat (dst, "[0-9]?");
582 for (i = 0; i <= pos; i++)
583 strcat (dst, "[0-9]");
584 strcat (dst, "[0-9]*");
589 static int relational_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
590 const char **term_sub,
592 oid_value attributeSet,
593 struct grep_info *grep_info,
601 attr_init (&relation, zapt, 2);
602 relation_value = attr_find (&relation, NULL);
604 switch (relation_value)
607 if (!term_100 (term_sub, term_dict, 1))
609 term_value = atoi (term_dict);
612 logf (LOG_DEBUG, "Relation <");
613 gen_regular_rel (term_dict + strlen(term_dict), term_value-1, 1);
616 if (!term_100 (term_sub, term_dict, 1))
618 term_value = atoi (term_dict);
621 logf (LOG_DEBUG, "Relation <=");
622 gen_regular_rel (term_dict + strlen(term_dict), term_value, 1);
625 if (!term_100 (term_sub, term_dict, 1))
627 term_value = atoi (term_dict);
630 logf (LOG_DEBUG, "Relation >=");
631 gen_regular_rel (term_dict + strlen(term_dict), term_value, 0);
634 if (!term_100 (term_sub, term_dict, 1))
636 term_value = atoi (term_dict);
639 logf (LOG_DEBUG, "Relation >");
640 gen_regular_rel (term_dict + strlen(term_dict), term_value+1, 0);
645 logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict);
646 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info, max_pos,
649 logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
650 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
654 static int field_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
655 const char **term_sub, int regType,
656 oid_value attributeSet, struct grep_info *grep_info,
657 int num_bases, char **basenames, int space_split)
659 char term_dict[2*IT_MAX_WORD+2];
662 int truncation_value;
665 oid_value curAttributeSet = attributeSet;
668 attr_init (&use, zapt, 1);
669 use_value = attr_find (&use, &curAttributeSet);
670 logf (LOG_DEBUG, "field_term, use value %d", use_value);
671 attr_init (&truncation, zapt, 5);
672 truncation_value = attr_find (&truncation, NULL);
673 logf (LOG_DEBUG, "truncation value %d", truncation_value);
678 for (base_no = 0; base_no < num_bases; base_no++)
681 data1_local_attribute *local_attr;
682 int max_pos, prefix_len = 0;
685 if (!att_getentbyatt (zi, &attp, curAttributeSet, use_value))
687 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
688 curAttributeSet, use_value);
692 if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
694 zi->errCode = 109; /* Database unavailable */
695 zi->errString = basenames[base_no];
698 for (local_attr = attp.local_attributes; local_attr;
699 local_attr = local_attr->next)
703 ord = zebTargetInfo_lookupSU (zi->zti, attp.attset_ordinal,
708 term_dict[prefix_len++] = '|';
710 term_dict[prefix_len++] = '(';
711 term_dict[prefix_len++] = 1;
712 term_dict[prefix_len++] = ord;
719 term_dict[prefix_len++] = ')';
720 term_dict[prefix_len++] = 1;
721 term_dict[prefix_len++] = regType;
722 term_dict[prefix_len] = '\0';
723 if (!relational_term (zi, zapt, &termp, term_dict,
724 attributeSet, grep_info, &max_pos))
727 switch (truncation_value)
729 case -1: /* not specified */
730 case 100: /* do not truncate */
731 term_dict[j++] = '(';
732 if (!term_100 (&termp, term_dict + j, space_split))
734 strcat (term_dict, ")");
735 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
736 &max_pos, 0, grep_handle);
738 logf (LOG_WARN, "dict_lookup_grep err, trunc=none:%d", r);
740 case 1: /* right truncation */
741 term_dict[j++] = '(';
742 if (!term_100 (&termp, term_dict + j, space_split))
744 strcat (term_dict, ".*)");
745 dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
746 &max_pos, 0, grep_handle);
748 case 2: /* left truncation */
749 case 3: /* left&right truncation */
752 case 101: /* process # in term */
753 term_dict[j++] = '(';
754 if (!term_101 (&termp, term_dict + j, space_split))
756 strcat (term_dict, ")");
757 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
758 &max_pos, 0, grep_handle);
760 logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d", r);
762 case 102: /* Regexp-1 */
763 term_dict[j++] = '(';
764 if (!term_102 (&termp, term_dict + j, space_split))
766 strcat (term_dict, ")");
767 logf (LOG_DEBUG, "Regexp-1 tolerance=%d", r);
768 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
769 &max_pos, 0, grep_handle);
771 logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
774 case 103: /* Regexp-1 */
776 term_dict[j++] = '(';
777 if (!term_103 (&termp, term_dict + j, &r, space_split))
779 strcat (term_dict, ")");
780 logf (LOG_DEBUG, "Regexp-2 tolerance=%d", r);
781 r = dict_lookup_grep (zi->dict, term_dict, r, grep_info,
782 &max_pos, 2, grep_handle);
784 logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
791 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
795 static void trans_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
799 Z_Term *term = zapt->term;
801 sizez = term->u.general->len;
802 if (sizez > IT_MAX_WORD-1)
803 sizez = IT_MAX_WORD-1;
804 memcpy (termz, term->u.general->buf, sizez);
808 static void trans_scan_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
811 Z_Term *term = zapt->term;
813 const char *cp = (const char *) term->u.general->buf;
814 const char *cp_end = cp + term->u.general->len;
817 const char *space_map = NULL;
820 while ((len = (cp_end - cp)) > 0)
822 map = map_chrs_input (&cp, len);
823 if (**map == *CHR_SPACE)
828 for (src = space_map; *src; src++)
831 for (src = *map; *src; src++)
838 static RSET rpn_search_APT_relevance (ZServerInfo *zi,
839 Z_AttributesPlusTerm *zapt,
840 oid_value attributeSet,
841 int num_bases, char **basenames)
843 rset_relevance_parms parms;
844 char termz[IT_MAX_WORD+1];
845 const char *termp = termz;
846 struct grep_info grep_info;
851 parms.key_size = sizeof(struct it_key);
852 parms.max_rec = 1000;
853 parms.cmp = key_compare_it;
855 parms.isc = zi->isamc;
858 if (zapt->term->which != Z_Term_general)
863 trans_term (zi, zapt, termz);
866 grep_info.term_no = 0;
868 grep_info.isam_p_indx = 0;
869 grep_info.isam_p_size = 0;
870 grep_info.isam_p_buf = NULL;
873 r = field_term (zi, zapt, &termp, 'w', attributeSet, &grep_info,
874 num_bases, basenames, 1);
878 for (; term_index < grep_info.isam_p_indx; term_index++)
879 grep_info.term_no[term_index] = parms.no_terms;
883 parms.term_no = grep_info.term_no;
884 parms.isam_positions = grep_info.isam_p_buf;
885 parms.no_isam_positions = grep_info.isam_p_indx;
886 if (grep_info.isam_p_indx > 0)
887 result = rset_create (rset_kind_relevance, &parms);
889 result = rset_create (rset_kind_null, NULL);
891 xfree(grep_info.term_no);
893 xfree (grep_info.isam_p_buf);
897 static RSET rpn_search_APT_cphrase (ZServerInfo *zi,
898 Z_AttributesPlusTerm *zapt,
899 oid_value attributeSet,
900 int num_bases, char **basenames)
902 char termz[IT_MAX_WORD+1];
903 struct grep_info grep_info;
905 const char *termp = termz;
908 if (zapt->term->which != Z_Term_general)
913 trans_term (zi, zapt, termz);
916 grep_info.term_no = 0;
918 grep_info.isam_p_indx = 0;
919 grep_info.isam_p_size = 0;
920 grep_info.isam_p_buf = NULL;
922 r = field_term (zi, zapt, &termp, 'p', attributeSet, &grep_info,
923 num_bases, basenames, 0);
924 result = rset_trunc (zi, grep_info.isam_p_buf, grep_info.isam_p_indx);
926 xfree(grep_info.term_no);
928 xfree (grep_info.isam_p_buf);
932 static RSET rpn_proximity (ZServerInfo *zi, RSET rset1, RSET rset2,
934 int exclusion, int relation, int distance)
939 struct it_key buf1, buf2;
942 rset_temp_parms parms;
944 rsfd1 = rset_open (rset1, RSETF_READ|RSETF_SORT_SYSNO);
945 more1 = rset_read (rset1, rsfd1, &buf1);
947 rsfd2 = rset_open (rset2, RSETF_READ|RSETF_SORT_SYSNO);
948 more2 = rset_read (rset2, rsfd2, &buf2);
950 parms.key_size = sizeof (struct it_key);
951 parms.temp_path = res_get (zi->res, "setTmpDir");
952 result = rset_create (rset_kind_temp, &parms);
953 rsfd_result = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
955 logf (LOG_DEBUG, "rpn_proximity excl=%d ord=%d rel=%d dis=%d",
956 exclusion, ordered, relation, distance);
957 while (more1 && more2)
959 int cmp = key_compare_it (&buf1, &buf2);
961 more1 = rset_read (rset1, rsfd1, &buf1);
963 more2 = rset_read (rset2, rsfd2, &buf2);
966 int sysno = buf1.sysno;
970 seqno[n++] = buf1.seqno;
971 while ((more1 = rset_read (rset1, rsfd1, &buf1)) &&
974 seqno[n++] = buf1.seqno;
977 for (i = 0; i<n; i++)
979 int diff = buf2.seqno - seqno[i];
980 int excl = exclusion;
981 if (!ordered && diff < 0)
990 if (diff <= distance)
994 if (diff == distance)
998 if (diff >= distance)
1002 if (diff > distance)
1006 if (diff != distance)
1011 rset_write (result, rsfd_result, &buf2);
1013 } while ((more2 = rset_read (rset2, rsfd2, &buf2)) &&
1014 sysno == buf2.sysno);
1017 rset_close (result, rsfd_result);
1018 rset_close (rset1, rsfd1);
1019 rset_close (rset2, rsfd2);
1023 static RSET rpn_prox (ZServerInfo *zi, RSET *rset, int rset_no)
1028 struct it_key **buf;
1031 rset_temp_parms parms;
1033 rsfd = xmalloc (sizeof(*rsfd)*rset_no);
1034 more = xmalloc (sizeof(*more)*rset_no);
1035 buf = xmalloc (sizeof(*buf)*rset_no);
1037 for (i = 0; i<rset_no; i++)
1039 buf[i] = xmalloc (sizeof(**buf));
1040 rsfd[i] = rset_open (rset[i], RSETF_READ|RSETF_SORT_SYSNO);
1041 if (!(more[i] = rset_read (rset[i], rsfd[i], buf[i])))
1045 rset_close (rset[i], rsfd[i]);
1052 return rset_create (rset_kind_null, NULL);
1055 parms.key_size = sizeof (struct it_key);
1056 parms.temp_path = res_get (zi->res, "setTmpDir");
1057 result = rset_create (rset_kind_temp, &parms);
1058 rsfd_result = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
1062 for (i = 1; i<rset_no; i++)
1071 cmp = key_compare_it (buf[i], buf[i-1]);
1074 more[i-1] = rset_read (rset[i-1], rsfd[i-1], buf[i-1]);
1079 if (buf[i-1]->seqno+1 != buf[i]->seqno)
1081 more[i-1] = rset_read (rset[i-1], rsfd[i-1], buf[i-1]);
1087 more[i] = rset_read (rset[i], rsfd[i], buf[i]);
1093 rset_write (result, rsfd_result, buf[0]);
1094 more[0] = rset_read (*rset, *rsfd, *buf);
1098 for (i = 0; i<rset_no; i++)
1100 rset_close (rset[i], rsfd[i]);
1103 rset_close (result, rsfd_result);
1110 static RSET rpn_search_APT_phrase (ZServerInfo *zi,
1111 Z_AttributesPlusTerm *zapt,
1112 oid_value attributeSet,
1113 int num_bases, char **basenames)
1115 char termz[IT_MAX_WORD+1];
1116 const char *termp = termz;
1117 RSET rset[60], result;
1118 int i, r, rset_no = 0;
1119 struct grep_info grep_info;
1121 if (zapt->term->which != Z_Term_general)
1126 trans_term (zi, zapt, termz);
1129 grep_info.term_no = 0;
1131 grep_info.isam_p_size = 0;
1132 grep_info.isam_p_buf = NULL;
1136 grep_info.isam_p_indx = 0;
1137 r = field_term (zi, zapt, &termp, 'w', attributeSet, &grep_info,
1138 num_bases, basenames, 1);
1141 rset[rset_no] = rset_trunc (zi, grep_info.isam_p_buf,
1142 grep_info.isam_p_indx);
1143 assert (rset[rset_no]);
1144 if (++rset_no >= sizeof(rset)/sizeof(*rset))
1148 xfree(grep_info.term_no);
1150 xfree (grep_info.isam_p_buf);
1152 return rset_create (rset_kind_null, NULL);
1153 else if (rset_no == 1)
1155 result = rpn_prox (zi, rset, rset_no);
1156 for (i = 0; i<rset_no; i++)
1157 rset_delete (rset[i]);
1161 static RSET rpn_search_APT_local (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1162 oid_value attributeSet)
1167 rset_temp_parms parms;
1168 char termz[IT_MAX_WORD+1];
1170 if (zapt->term->which != Z_Term_general)
1175 parms.key_size = sizeof (struct it_key);
1176 parms.temp_path = res_get (zi->res, "setTmpDir");
1177 result = rset_create (rset_kind_temp, &parms);
1178 rsfd = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
1180 trans_term (zi, zapt, termz);
1182 key.sysno = atoi (termz);
1185 rset_write (result, rsfd, &key);
1186 rset_close (result, rsfd);
1190 static RSET rpn_search_APT (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1191 oid_value attributeSet,
1192 int num_bases, char **basenames)
1196 AttrType completeness;
1197 int relation_value, structure_value, completeness_value;
1199 attr_init (&relation, zapt, 2);
1200 attr_init (&structure, zapt, 4);
1201 attr_init (&completeness, zapt, 6);
1203 relation_value = attr_find (&relation, NULL);
1204 structure_value = attr_find (&structure, NULL);
1205 completeness_value = attr_find (&completeness, NULL);
1206 switch (structure_value)
1209 if (relation_value == 102) /* relevance relation */
1210 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1211 num_bases, basenames);
1212 if (completeness_value == 2 || completeness_value == 3)
1213 return rpn_search_APT_cphrase (zi, zapt, attributeSet,
1214 num_bases, basenames);
1215 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1216 num_bases, basenames);
1217 case 1: /* phrase */
1218 if (relation_value == 102) /* relevance relation */
1219 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1220 num_bases, basenames);
1221 if (completeness_value == 2 || completeness_value == 3)
1222 return rpn_search_APT_cphrase (zi, zapt, attributeSet,
1223 num_bases, basenames);
1224 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1225 num_bases, basenames);
1228 if (relation_value == 102) /* relevance relation */
1229 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1230 num_bases, basenames);
1231 if (completeness_value == 2 || completeness_value == 3)
1232 return rpn_search_APT_cphrase (zi, zapt, attributeSet,
1233 num_bases, basenames);
1234 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1235 num_bases, basenames);
1240 case 5: /* date - normalized */
1242 case 6: /* word list */
1243 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1244 num_bases, basenames);
1245 case 100: /* date - un-normalized */
1247 case 101: /* name - normalized */
1249 case 102: /* date - un-normalized */
1251 case 103: /* structure */
1255 case 105: /* free-form-text */
1256 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1257 num_bases, basenames);
1258 case 106: /* document-text */
1259 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1260 num_bases, basenames);
1261 case 107: /* local-number */
1262 return rpn_search_APT_local (zi, zapt, attributeSet);
1263 case 108: /* string */
1264 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1265 num_bases, basenames);
1266 case 109: /* numeric string */
1273 static RSET rpn_search_ref (ZServerInfo *zi, Z_ResultSetId *resultSetId)
1277 if (!(s = resultSetGet (zi, resultSetId)))
1278 return rset_create (rset_kind_null, NULL);
1282 static RSET rpn_search_structure (ZServerInfo *zi, Z_RPNStructure *zs,
1283 oid_value attributeSet,
1284 int num_bases, char **basenames)
1287 if (zs->which == Z_RPNStructure_complex)
1289 Z_Operator *zop = zs->u.complex->roperator;
1290 rset_bool_parms bool_parms;
1294 bool_parms.rset_l = rpn_search_structure (zi, zs->u.complex->s1,
1296 num_bases, basenames);
1297 if (bool_parms.rset_l == NULL)
1299 if (rset_is_ranked(bool_parms.rset_l))
1301 bool_parms.rset_r = rpn_search_structure (zi, zs->u.complex->s2,
1303 num_bases, basenames);
1304 if (bool_parms.rset_r == NULL)
1306 rset_delete (bool_parms.rset_l);
1309 if (rset_is_ranked(bool_parms.rset_r))
1311 bool_parms.key_size = sizeof(struct it_key);
1312 bool_parms.cmp = key_compare_it;
1316 case Z_Operator_and:
1317 r = rset_create (soft ? rset_kind_sand:rset_kind_and, &bool_parms);
1320 r = rset_create (soft ? rset_kind_sor:rset_kind_or, &bool_parms);
1322 case Z_Operator_and_not:
1323 r = rset_create (soft ? rset_kind_snot:rset_kind_not, &bool_parms);
1325 case Z_Operator_prox:
1326 if (zop->u.prox->which != Z_ProxCode_known)
1331 if (*zop->u.prox->proximityUnitCode != Z_ProxUnit_word)
1333 static char val[16];
1335 zi->errString = val;
1336 sprintf (val, "%d", *zop->u.prox->proximityUnitCode);
1339 r = rpn_proximity (zi, bool_parms.rset_l, bool_parms.rset_r,
1340 *zop->u.prox->ordered,
1341 (!zop->u.prox->exclusion ? 0 :
1342 *zop->u.prox->exclusion),
1343 *zop->u.prox->relationType,
1344 *zop->u.prox->distance);
1351 else if (zs->which == Z_RPNStructure_simple)
1353 if (zs->u.simple->which == Z_Operand_APT)
1355 logf (LOG_DEBUG, "rpn_search_APT");
1356 r = rpn_search_APT (zi, zs->u.simple->u.attributesPlusTerm,
1357 attributeSet, num_bases, basenames);
1359 else if (zs->u.simple->which == Z_Operand_resultSetId)
1361 logf (LOG_DEBUG, "rpn_search_ref");
1362 r = rpn_search_ref (zi, zs->u.simple->u.resultSetId);
1378 void count_set_save (ZServerInfo *zi, RSET *r, int *count)
1385 rset_temp_parms parms;
1386 int maxResultSetSize = atoi (res_get_def (zi->res,
1387 "maxResultSetSize", "400"));
1388 logf (LOG_DEBUG, "count_set_save");
1390 parms.key_size = sizeof(struct it_key);
1391 parms.temp_path = res_get (zi->res, "setTmpDir");
1392 w = rset_create (rset_kind_temp, &parms);
1393 wfd = rset_open (w, RSETF_WRITE|RSETF_SORT_SYSNO);
1394 rfd = rset_open (*r, RSETF_READ|RSETF_SORT_SYSNO);
1395 while (rset_read (*r, rfd, &key))
1397 if (key.sysno != psysno)
1399 if (*count < maxResultSetSize)
1400 rset_write (w, wfd, &key);
1406 rset_close (*r, rfd);
1408 rset_close (w, wfd);
1410 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
1413 static void count_set (RSET r, int *count)
1420 logf (LOG_DEBUG, "count_set");
1422 rfd = rset_open (r, RSETF_READ|RSETF_SORT_SYSNO);
1423 while (rset_read (r, rfd, &key))
1425 if (key.sysno != psysno)
1432 rset_close (r, rfd);
1433 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
1436 int rpn_search (ZServerInfo *zi,
1437 Z_RPNQuery *rpn, int num_bases, char **basenames,
1438 const char *setname, int *hits)
1442 oid_value attributeSet;
1444 dict_grep_cmap (zi->dict, map_chrs_input);
1448 zi->errString = NULL;
1450 attrset = oid_getentbyoid (rpn->attributeSetId);
1451 attributeSet = attrset->value;
1452 rset = rpn_search_structure (zi, rpn->RPNStructure, attributeSet,
1453 num_bases, basenames);
1456 if (rset_is_volatile(rset))
1457 count_set_save(zi, &rset,hits);
1459 count_set (rset, hits);
1460 resultSetAdd (zi, setname, 1, rset);
1462 logf (LOG_DEBUG, "search error: %d", zi->errCode);
1466 struct scan_info_entry {
1472 struct scan_info_entry *list;
1478 static int scan_handle (char *name, const char *info, int pos, void *client)
1480 int len_prefix, idx;
1481 struct scan_info *scan_info = client;
1483 len_prefix = strlen(scan_info->prefix);
1484 if (memcmp (name, scan_info->prefix, len_prefix))
1487 idx = scan_info->after - pos + scan_info->before;
1490 scan_info->list[idx].term = odr_malloc (scan_info->odr,
1491 strlen(name + len_prefix)+1);
1492 strcpy (scan_info->list[idx].term, name + len_prefix);
1493 assert (*info == sizeof(ISAM_P));
1494 memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
1499 static void scan_term_untrans (ODR odr, char **dstp, const char *src)
1501 char *dst = odr_malloc (odr, strlen(src)*2+1);
1506 const char *cp = map_chrs_output (&src);
1513 int rpn_scan (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1514 oid_value attributeset,
1515 int num_bases, char **basenames,
1516 int *position, int *num_entries, struct scan_entry **list,
1520 int pos = *position;
1521 int num = *num_entries;
1525 char termz[IT_MAX_WORD+20];
1528 AttrType completeness;
1529 int completeness_value;
1530 struct scan_info *scan_info_array;
1531 struct scan_entry *glist;
1532 int ords[32], ord_no = 0;
1535 logf (LOG_DEBUG, "scan, position = %d, num = %d", pos, num);
1537 if (attributeset == VAL_NONE)
1538 attributeset = VAL_BIB1;
1540 attr_init (&use, zapt, 1);
1541 use_value = attr_find (&use, &attributeset);
1542 logf (LOG_DEBUG, "use value %d", use_value);
1544 attr_init (&completeness, zapt, 6);
1545 completeness_value = attr_find (&completeness, NULL);
1546 logf (LOG_DEBUG, "completeness value %d", completeness_value);
1548 if (use_value == -1)
1550 for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
1553 data1_local_attribute *local_attr;
1555 if (!att_getentbyatt (zi, &attp, attributeset, use_value))
1557 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
1558 attributeset, use_value);
1559 return zi->errCode = 114;
1561 if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
1563 zi->errString = basenames[base_no];
1564 return zi->errCode = 109; /* Database unavailable */
1566 for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
1567 local_attr = local_attr->next)
1571 ord = zebTargetInfo_lookupSU (zi->zti, attp.attset_ordinal,
1574 ords[ord_no++] = ord;
1578 return zi->errCode = 113;
1581 scan_info_array = odr_malloc (zi->odr, ord_no * sizeof(*scan_info_array));
1582 for (i = 0; i < ord_no; i++)
1584 int j, prefix_len = 0;
1585 int before_tmp = before, after_tmp = after;
1586 struct scan_info *scan_info = scan_info_array + i;
1588 scan_info->before = before;
1589 scan_info->after = after;
1590 scan_info->odr = zi->odr;
1592 scan_info->list = odr_malloc (zi->odr, (before+after)*
1593 sizeof(*scan_info->list));
1594 for (j = 0; j<before+after; j++)
1595 scan_info->list[j].term = NULL;
1596 termz[prefix_len++] = ords[i];
1597 termz[prefix_len++] =
1598 (completeness_value==2 || completeness_value==3) ? 'p': 'w';
1599 termz[prefix_len] = 0;
1600 strcpy (scan_info->prefix, termz);
1602 trans_scan_term (zi, zapt, termz+prefix_len);
1604 dict_scan (zi->dict, termz, &before_tmp, &after_tmp, scan_info,
1607 glist = odr_malloc (zi->odr, (before+after)*sizeof(*glist));
1608 for (i = 0; i < ord_no; i++)
1611 *status = BEND_SCAN_SUCCESS;
1612 for (i = 0; i<after; i++)
1615 const char *mterm = NULL;
1619 for (j = 0; j < ord_no; j++)
1621 if (ptr[j] < before+after &&
1622 (tst=scan_info_array[j].list[ptr[j]].term) &&
1623 (!mterm || strcmp (tst, mterm) < 0))
1631 scan_term_untrans (zi->odr, &glist[i+before].term, mterm);
1632 rset = rset_trunc (zi, &scan_info_array[j0].list[ptr[j0]].isam_p, 1);
1635 for (j = j0+1; j<ord_no; j++)
1637 if (ptr[j] < before+after &&
1638 (tst=scan_info_array[j].list[ptr[j]].term) &&
1639 !strcmp (tst, mterm))
1641 rset_bool_parms bool_parms;
1645 rset_trunc (zi, &scan_info_array[j].list[ptr[j]].isam_p, 1);
1647 bool_parms.key_size = sizeof(struct it_key);
1648 bool_parms.cmp = key_compare_it;
1649 bool_parms.rset_l = rset;
1650 bool_parms.rset_r = rset2;
1652 rset = rset_create (rset_kind_or, &bool_parms);
1657 count_set (rset, &glist[i+before].occurrences);
1662 *num_entries -= (after-i);
1663 *status = BEND_SCAN_PARTIAL;
1666 for (i = 0; i<ord_no; i++)
1669 for (i = 0; i<before; i++)
1672 const char *mterm = NULL;
1676 for (j = 0; j <ord_no; j++)
1678 if (ptr[j] < before &&
1679 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
1680 (!mterm || strcmp (tst, mterm) > 0))
1689 scan_term_untrans (zi->odr, &glist[before-1-i].term, mterm);
1692 (zi, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1);
1696 for (j = j0+1; j<ord_no; j++)
1698 if (ptr[j] < before &&
1699 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
1700 !strcmp (tst, mterm))
1702 rset_bool_parms bool_parms;
1705 rset2 = rset_trunc (zi,
1706 &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1);
1708 bool_parms.key_size = sizeof(struct it_key);
1709 bool_parms.cmp = key_compare_it;
1710 bool_parms.rset_l = rset;
1711 bool_parms.rset_r = rset2;
1713 rset = rset_create (rset_kind_or, &bool_parms);
1718 count_set (rset, &glist[before-1-i].occurrences);
1724 *status = BEND_SCAN_PARTIAL;
1728 *list = glist + i; /* list is set to first 'real' entry */
1730 logf (LOG_DEBUG, "position = %d, num_entries = %d",
1731 *position, *num_entries);
1733 logf (LOG_DEBUG, "scan error: %d", zi->errCode);