1 /* $Id: benchindex1.c,v 1.6 2006-12-12 15:23:45 adam Exp $
2 Copyright (C) 1995-2006
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <yaz/options.h>
29 #include <yaz/xmalloc.h>
30 #include <yaz/marcdisp.h>
32 #include <idzebra/isamb.h>
33 #include <idzebra/dict.h>
41 struct index_term *terms;
42 struct index_term **ar;
51 struct index_term *next;
54 struct index_block *index_block_new(int memory)
56 struct index_block *b = xmalloc(sizeof(*b));
58 b->current_max = memory * 1024 * 1024;
60 b->nmem = nmem_create();
65 void index_block_destroy(struct index_block **bp)
69 nmem_destroy((*bp)->nmem);
75 static int cmp_ar(const void *p1, const void *p2)
77 struct index_term *t1 = *(struct index_term **) p1;
78 struct index_term *t2 = *(struct index_term **) p2;
79 int d = strcmp(t1->term, t2->term);
83 if (t1->docid > t2->docid)
85 else if (t1->docid < t2->docid)
87 if (t1->seqno > t2->seqno)
89 else if (t1->seqno < t2->seqno)
95 int code_read(void *vp, char **dst, int *insertMode)
97 struct index_block *b = (struct index_block *)vp;
101 if (b->current_entry >= b->no_entries)
104 t = b->ar[b->current_entry];
108 key.mem[0] = t->word_id;
109 key.mem[1] = t->docid;
110 key.mem[2] = t->seqno;
113 memcpy(*dst, &key, sizeof(key));
115 (*dst) += sizeof(key);
118 yaz_log(YLOG_LOG, "returning " ZINT_FORMAT " " ZINT_FORMAT "\n",
119 key.mem[0], key.mem[1]);
124 void index_block_flush(struct index_block *b, ISAMB isb, Dict dict,
127 struct index_term *t = b->terms;
130 int no_words = 0, no_new_words = 0;
131 const char *dict_info = 0;
133 zebra_timing_t tim_dict = 0;
134 zebra_timing_t tim_isamb = 0;
135 zint number_of_int_splits = isamb_get_int_splits(isb);
136 zint number_of_leaf_splits = isamb_get_leaf_splits(isb);
137 zint number_of_dict_splits = dict_get_no_split(dict);
139 b->ar = xmalloc(sizeof(*b->ar) * b->no_entries);
140 for (i = 0; i < b->no_entries; i++, t = t->next)
147 qsort(b->ar, b->no_entries, sizeof(*b->ar), cmp_ar);
148 tim_dict = zebra_timing_create();
150 for (i = 0; i < b->no_entries; i++)
152 printf("%s " ZINT_FORMAT " " ZINT_FORMAT "\n",
153 ar[i]->term, ar[i]->docid, ar[i]->seqno);
156 dict_info = dict_lookup(dict, "_w");
159 assert(*dict_info == sizeof(word_id_seq));
160 memcpy(&word_id_seq, dict_info+1, sizeof(word_id_seq));
163 dict_info = dict_lookup(dict, "_i");
166 assert(*dict_info == sizeof(isamc_p));
167 memcpy(&isamc_p, dict_info+1, sizeof(isamc_p));
170 for (i = 0; i < b->no_entries; i++)
172 if (i > 0 && strcmp(b->ar[i-1]->term, b->ar[i]->term) == 0)
173 b->ar[i]->word_id = b->ar[i-1]->word_id;
176 const char *dict_info = dict_lookup(dict, b->ar[i]->term);
179 memcpy(&b->ar[i]->word_id, dict_info+1, sizeof(int));
185 dict_insert(dict, b->ar[i]->term, sizeof(int), &word_id_seq);
186 b->ar[i]->word_id = word_id_seq;
191 dict_insert(dict, "_w", sizeof(word_id_seq), &word_id_seq);
193 zebra_timing_stop(tim_dict);
194 tim_isamb = zebra_timing_create();
196 b->current_entry = 0;
202 isamc_i.clientData = b;
203 isamc_i.read_item = code_read;
205 isamb_merge (isb, &isamc_p, &isamc_i);
208 dict_insert(dict, "_i", sizeof(isamc_p), &isamc_p);
211 zebra_timing_stop(tim_isamb);
213 number_of_int_splits = isamb_get_int_splits(isb) - number_of_int_splits;
214 number_of_leaf_splits = isamb_get_leaf_splits(isb) - number_of_leaf_splits;
215 number_of_dict_splits = dict_get_no_split(dict) - number_of_dict_splits;
219 printf("# run total dict-real user sys isam-real user sys "
220 " intsp leafsp docs postings words new d-spl\n");
223 printf("%5d %9.6f %9.6f %5.2f %5.2f %9.6f %5.2f %5.2f "
224 "%6" ZINT_FORMAT0 " %6" ZINT_FORMAT0
225 " %5d %8d %6d %6d" " %5" ZINT_FORMAT0 "\n",
227 zebra_timing_get_real(tim_dict) + zebra_timing_get_real(tim_isamb),
228 zebra_timing_get_real(tim_dict),
229 zebra_timing_get_user(tim_dict),
230 zebra_timing_get_sys(tim_dict),
231 zebra_timing_get_real(tim_isamb),
232 zebra_timing_get_user(tim_isamb),
233 zebra_timing_get_sys(tim_isamb),
234 number_of_int_splits,
235 number_of_leaf_splits,
240 number_of_dict_splits
250 zebra_timing_destroy(&tim_isamb);
251 zebra_timing_destroy(&tim_dict);
254 void index_block_check_flush(struct index_block *b, ISAMB isb, Dict dict,
257 int total = nmem_total(b->nmem);
258 int max = b->current_max;
261 index_block_flush(b, isb, dict, no_docs);
265 void index_block_add(struct index_block *b,
266 const char *term, zint docid, zint seqno)
268 struct index_term *t = nmem_malloc(b->nmem, sizeof(*t));
269 t->term = nmem_strdup(b->nmem, term);
277 void index_term(struct index_block *b, const char *term,
278 zint docid, zint *seqno)
281 printf("%s " ZINT_FORMAT " " ZINT_FORMAT "\n", term,
284 index_block_add(b, term, docid, *seqno);
288 void index_wrbuf(struct index_block *b, WRBUF wrbuf, zint docid,
292 const char *cp = wrbuf_buf(wrbuf);
303 { /* skip field+indicator (e.g. 245 00) */
304 for (i = 0; i<6 && *cp; i++, cp++)
308 { /* continuation line */
309 for (i = 0; i<4 && *cp; i++, cp++)
318 index_term(b, term, docid, &seqno);
324 else if (*cp == subfield_char && cp[1])
328 index_term(b, term, docid, &seqno);
333 else if (strchr("$*/-;,.:[]\"&(){} ", *cp))
337 index_term(b, term, docid, &seqno);
344 unsigned ch = *(const unsigned char *)cp;
345 if (sz < sizeof(term))
347 term[sz] = tolower(ch);
355 index_term(b, term, docid, &seqno);
358 void index_marc_line_records(ISAMB isb,
364 WRBUF wrbuf = wrbuf_alloc();
368 struct index_block *b = index_block_new(memory);
369 while(fgets(line, sizeof(line)-1, inf))
383 index_block_check_flush(b, isb, dict, no_docs);
390 wrbuf_puts(wrbuf, line);
395 /* index existing buffer (if any) */
396 if (wrbuf_len(wrbuf))
398 index_wrbuf(b, wrbuf, *docid_seq, '*');
401 if (line[0] != ' ' && line[1] != ' ' && line[2] != ' ' &&
404 /* normal field+indicator line */
405 wrbuf_puts(wrbuf, line);
409 if (wrbuf_len(wrbuf))
411 index_wrbuf(b, wrbuf, *docid_seq, '*');
416 index_block_flush(b, isb, dict, no_docs);
417 index_block_destroy(&b);
420 void index_marc_from_file(ISAMB isb,
425 int verbose, int print_offset)
427 yaz_marc_t mt = yaz_marc_create();
428 WRBUF wrbuf = wrbuf_alloc();
429 struct index_block *b = index_block_new(memory);
438 r = fread (buf, 1, 5, inf);
441 if (r && print_offset && verbose)
442 printf ("<!-- Extra %ld bytes at end of file -->\n",
446 while (*buf < '0' || *buf > '9')
449 long off = ftell(inf) - 5;
450 if (verbose || print_offset)
451 printf("<!-- Skipping bad byte %d (0x%02X) at offset "
453 *buf & 0xff, *buf & 0xff,
455 for (i = 0; i<4; i++)
457 r = fread(buf+4, 1, 1, inf);
463 if (verbose || print_offset)
464 printf ("<!-- End of file with data -->\n");
467 len = atoi_n(buf, 5);
468 if (len < 25 || len > 100000)
470 long off = ftell(inf) - 5;
471 printf("Bad Length %ld read at offset %ld (%lx)\n",
472 (long)len, (long) off, (long) off);
476 r = fread (buf + 5, 1, rlen, inf);
479 yaz_marc_read_iso2709(mt, buf, len);
481 if (yaz_marc_write_line(mt, wrbuf))
484 index_wrbuf(b, wrbuf, *docid_seq, '$');
489 index_block_check_flush(b, isb, dict, no_docs);
491 index_block_flush(b, isb, dict, no_docs);
492 wrbuf_free(wrbuf, 1);
493 yaz_marc_destroy(mt);
494 index_block_destroy(&b);
497 void exit_usage(void)
499 fprintf(stderr, "benchindex1 [-t type] [-c d:i] [-m mem] [-i] [inputfile]\n");
503 int main(int argc, char **argv)
507 ISAMC_M method_postings;
513 int isam_cache_size = 40;
514 int dict_cache_size = 50;
515 const char *fname = 0;
517 zebra_timing_t tim = 0;
519 const char *dict_info;
520 const char *type = "iso2709";
521 int int_count_enable = 1;
523 while ((ret = options("im:t:c:N", argv, argc, &arg)) != -2)
534 if (!strcmp(arg, "iso2709"))
536 else if (!strcmp(arg, "line"))
540 fprintf(stderr, "bad type: %s.\n", arg);
545 if (sscanf(arg, "%d:%d", &dict_cache_size, &isam_cache_size)
548 fprintf(stderr, "bad cache sizes for -c\n");
556 int_count_enable = 0;
559 fprintf(stderr, "bad option.\n");
566 inf = fopen(fname, "rb");
569 fprintf(stderr, "Cannot open %s\n", fname);
573 printf("# benchindex1 %s %s\n", __DATE__, __TIME__);
574 printf("# isam_cache_size = %d\n", isam_cache_size);
575 printf("# dict_cache_size = %d\n", dict_cache_size);
576 printf("# int_count_enable = %d\n", int_count_enable);
577 printf("# memory = %d\n", memory);
579 /* setup postings isamb attributes */
580 method_postings.compare_item = key_compare;
581 method_postings.log_item = key_logdump_txt;
583 method_postings.codec.start = iscz1_start;
584 method_postings.codec.decode = iscz1_decode;
585 method_postings.codec.encode = iscz1_encode;
586 method_postings.codec.stop = iscz1_stop;
587 method_postings.codec.reset = iscz1_reset;
589 method_postings.debug = 0;
591 /* create block system */
592 bfs = bfs_create(0, 0);
595 yaz_log(YLOG_WARN, "bfs_create failed");
602 tim = zebra_timing_create();
603 /* create isam handle */
604 isb_postings = isamb_open (bfs, "isamb", isam_cache_size ? 1 : 0,
605 &method_postings, 0);
608 yaz_log(YLOG_WARN, "isamb_open failed");
611 isamb_set_cache_size(isb_postings, isam_cache_size);
612 isamb_set_int_count(isb_postings, int_count_enable);
613 dict = dict_open(bfs, "dict", dict_cache_size, 1, 0, 4096);
615 dict_info = dict_lookup(dict, "_s");
618 assert(*dict_info == sizeof(docid_seq));
619 memcpy(&docid_seq, dict_info+1, sizeof(docid_seq));
622 if (!strcmp(type, "iso2709"))
623 index_marc_from_file(isb_postings, dict, &docid_seq, inf, memory,
624 0 /* verbose */ , 0 /* print_offset */);
625 else if (!strcmp(type, "line"))
626 index_marc_line_records(isb_postings, dict, &docid_seq, inf, memory);
628 printf("# Total " ZINT_FORMAT " documents\n", docid_seq);
629 dict_insert(dict, "_s", sizeof(docid_seq), &docid_seq);
632 isamb_close(isb_postings);
636 /* exit block system */
638 zebra_timing_stop(tim);
640 printf("# Total timings real=%8.6f user=%3.2f system=%3.2f\n",
641 zebra_timing_get_real(tim),
642 zebra_timing_get_user(tim),
643 zebra_timing_get_sys(tim));
645 zebra_timing_destroy(&tim);
653 * indent-tabs-mode: nil
655 * vim: shiftwidth=4 tabstop=8 expandtab