1 /* This file is part of the Zebra server.
2 Copyright (C) Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
37 #define KEY_SIZE (1+sizeof(struct it_key))
38 #define INP_NAME_MAX 768
42 off_t offset; /* file offset */
43 unsigned char *buf; /* buffer block */
44 size_t buf_size; /* number of read bytes in block */
45 size_t chunk; /* number of bytes allocated */
46 size_t buf_ptr; /* current position in buffer */
47 char *prev_name; /* last word read */
49 off_t length; /* length of file */
50 /* handler invoked in each read */
51 void (*readHandler)(struct key_file *keyp, void *rinfo);
60 #if PR_KEY_LOW || PR_KEY_TOP
62 static void pkey(const char *b, int mode)
64 key_logdump_txt(YLOG_LOG, b, mode ? "i" : "d");
69 void getFnameTmp(Res res, char *fname, int no)
73 pre = res_get_def(res, "keyTmpDir", ".");
74 sprintf(fname, "%s/key%d.tmp", pre, no);
77 void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no)
81 pre = res_get_def(zh->res, "keyTmpDir", ".");
82 sprintf(fname, "%s/key%d.tmp", pre, no);
85 void key_file_chunk_read(struct key_file *f)
87 int nr = 0, r = 0, fd;
89 getFnameTmp(f->res, fname, f->no);
90 fd = open(fname, O_BINARY|O_RDONLY);
96 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot open %s", fname);
101 if ((f->length = lseek(fd, 0L, SEEK_END)) == (off_t) -1)
103 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
108 if (lseek(fd, f->offset, SEEK_SET) == -1)
110 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
114 while (f->chunk - nr > 0)
116 r = read(fd, f->buf + nr, f->chunk - nr);
123 yaz_log(YLOG_WARN|YLOG_ERRNO, "read of %s", fname);
129 (*f->readHandler)(f, f->readInfo);
133 void key_file_destroy(struct key_file *f)
135 iscz1_stop(f->decode_handle);
141 struct key_file *key_file_init(int no, int chunk, Res res)
145 f = (struct key_file *) xmalloc(sizeof(*f));
147 f->decode_handle = iscz1_start();
152 f->readHandler = NULL;
153 f->buf = (unsigned char *) xmalloc(f->chunk);
154 f->prev_name = (char *) xmalloc(INP_NAME_MAX);
155 *f->prev_name = '\0';
156 key_file_chunk_read(f);
160 int key_file_getc(struct key_file *f)
162 if (f->buf_ptr < f->buf_size)
163 return f->buf[(f->buf_ptr)++];
164 if (f->buf_size < f->chunk)
166 f->offset += f->buf_size;
167 key_file_chunk_read(f);
168 if (f->buf_ptr < f->buf_size)
169 return f->buf[(f->buf_ptr)++];
174 int key_file_read(struct key_file *f, char *key)
178 const char *src = srcbuf;
182 c = key_file_getc(f);
185 strcpy(key, f->prev_name);
194 while ((c = key_file_getc(f)))
196 if (i < INP_NAME_MAX-2)
200 strcpy(f->prev_name, key);
201 iscz1_reset(f->decode_handle);
203 c = key_file_getc(f); /* length + insert/delete combined */
206 for (j = 0; j < c; j++)
207 srcbuf[j] = key_file_getc(f);
209 iscz1_decode(f->decode_handle, &dst, &src);
216 memcpy(&k, key+i, sizeof(k));
218 yaz_log(YLOG_LOG, "00 KEY");
221 return i + sizeof(struct it_key);
226 struct key_file **file;
231 int (*cmp)(const void *p1, const void *p2);
232 struct zebra_register *reg;
241 static struct heap_info *key_heap_malloc(void)
242 { /* malloc and clear it */
243 struct heap_info *hi;
244 hi = (struct heap_info *) xmalloc(sizeof(*hi));
252 hi->no_deletions = 0;
253 hi->no_insertions = 0;
254 hi->no_iterations = 0;
258 struct heap_info *key_heap_init_file(ZebraHandle zh,
260 int (*cmp)(const void *p1, const void *p2))
262 struct heap_info *hi;
265 hi = key_heap_malloc();
267 hi->info.file = (struct key_file **)
268 xmalloc(sizeof(*hi->info.file) * (1+nkeys));
269 hi->info.buf = (char **) xmalloc(sizeof(*hi->info.buf) * (1+nkeys));
270 hi->ptr = (int *) xmalloc(sizeof(*hi->ptr) * (1+nkeys));
272 for (i = 0; i<= nkeys; i++)
275 hi->info.buf[i] = (char *) xmalloc(INP_NAME_MAX);
280 void key_heap_destroy(struct heap_info *hi, int nkeys)
283 for (i = 0; i<=nkeys; i++)
284 xfree(hi->info.buf[i]);
287 xfree(hi->info.file);
291 static void key_heap_swap(struct heap_info *hi, int i1, int i2)
296 hi->ptr[i1] = hi->ptr[i2];
301 static void key_heap_delete(struct heap_info *hi)
303 int cur = 1, child = 2;
305 assert(hi->heapnum > 0);
307 key_heap_swap(hi, 1, hi->heapnum);
309 while (child <= hi->heapnum) {
310 if (child < hi->heapnum &&
311 (*hi->cmp)(&hi->info.buf[hi->ptr[child]],
312 &hi->info.buf[hi->ptr[child+1]]) > 0)
314 if ((*hi->cmp)(&hi->info.buf[hi->ptr[cur]],
315 &hi->info.buf[hi->ptr[child]]) > 0)
317 key_heap_swap(hi, cur, child);
326 static void key_heap_insert(struct heap_info *hi, const char *buf, int nbytes,
331 assert(nbytes < INP_NAME_MAX);
333 cur = ++(hi->heapnum);
334 memcpy(hi->info.buf[hi->ptr[cur]], buf, nbytes);
335 hi->info.file[hi->ptr[cur]] = kf;
338 while (parent && (*hi->cmp)(&hi->info.buf[hi->ptr[parent]],
339 &hi->info.buf[hi->ptr[cur]]) > 0)
341 key_heap_swap(hi, cur, parent);
347 static int heap_read_one(struct heap_info *hi, char *name, char *key)
350 char rbuf[INP_NAME_MAX];
356 strcpy(name, hi->info.buf[n]);
357 kf = hi->info.file[n];
359 memcpy(key, hi->info.buf[n] + r+1, KEY_SIZE);
361 if ((r = key_file_read(kf, rbuf)))
362 key_heap_insert(hi, rbuf, r, kf);
368 /* for debugging only */
369 void zebra_log_dict_entry(ZebraHandle zh, const char *s)
371 char dst[INP_NAME_MAX+1];
373 int len = key_SU_decode(&ord, (const unsigned char *) s);
374 const char *index_type;
377 yaz_log(YLOG_LOG, "ord=%d", ord);
380 const char *string_index;
382 zebraExplain_lookup_ord(zh->reg->zei,
383 ord, &index_type, &db, &string_index);
385 zebra_term_untrans(zh, index_type, dst, s + len);
387 yaz_log(YLOG_LOG, "ord=%d index_type=%s index=%s term=%s",
388 ord, index_type, string_index, dst);
392 struct heap_cread_info {
393 char prev_name[INP_NAME_MAX];
394 char cur_name[INP_NAME_MAX];
399 struct heap_info *hi;
406 static int heap_cread_item(void *vp, char **dst, int *insertMode);
408 int heap_cread_item2(void *vp, char **dst, int *insertMode)
410 struct heap_cread_info *p = (struct heap_cread_info *) vp;
415 if (p->look_level > 0)
425 memcpy(*dst, p->key_1, p->sz_1);
427 yaz_log(YLOG_LOG, "DUP level=%d", p->look_level);
428 pkey(*dst, *insertMode);
433 if (p->ret == 0) /* lookahead was 0?. Return that in read next round */
438 else if (p->ret == -1) /* Must read new item ? */
440 char *dst_1 = p->key_1;
441 p->ret = heap_cread_item(vp, &dst_1, &p->mode_1);
442 p->sz_1 = dst_1 - p->key_1;
445 { /* lookahead in 2 . Now in 1. */
447 p->mode_1 = p->mode_2;
448 memcpy(p->key_1, p->key_2, p->sz_2);
451 level = 1; /* insert */
453 level = -1; /* delete */
456 char *dst_2 = p->key_2;
457 p->ret = heap_cread_item(vp, &dst_2, &p->mode_2);
465 p->sz_2 = dst_2 - p->key_2;
467 if (key_compare(p->key_1, p->key_2) == 0)
469 if (p->mode_2) /* adjust level according to deletes/inserts */
478 /* all the same. new round .. */
480 p->mode_1 = p->mode_2;
481 memcpy(p->key_1, p->key_2, p->sz_1);
483 level = 1; /* insert */
485 level = -1; /* delete */
488 /* outcome is insert (1) or delete (0) depending on final level */
499 p->look_level = level;
500 memcpy(*dst, p->key_1, p->sz_1);
502 yaz_log(YLOG_LOG, "TOP");
503 pkey(*dst, *insertMode);
509 int heap_cread_item(void *vp, char **dst, int *insertMode)
511 struct heap_cread_info *p = (struct heap_cread_info *) vp;
512 struct heap_info *hi = p->hi;
514 if (p->first_in_list)
516 *insertMode = p->key[0];
517 memcpy(*dst, p->key+1, sizeof(struct it_key));
519 zebra_log_dict_entry(hi->zh, p->cur_name);
520 pkey(*dst, *insertMode);
522 (*dst) += sizeof(struct it_key);
523 p->first_in_list = 0;
526 strcpy(p->prev_name, p->cur_name);
527 if (!(p->more = heap_read_one(hi, p->cur_name, p->key)))
529 if (*p->cur_name && strcmp(p->cur_name, p->prev_name))
531 p->first_in_list = 1;
534 *insertMode = p->key[0];
535 memcpy(*dst, p->key+1, sizeof(struct it_key));
537 zebra_log_dict_entry(hi->zh, p->cur_name);
538 pkey(*dst, *insertMode);
540 (*dst) += sizeof(struct it_key);
544 int heap_inpc(struct heap_cread_info *hci, struct heap_info *hi)
546 ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
548 isamc_i->clientData = hci;
549 isamc_i->read_item = heap_cread_item2;
553 char this_name[INP_NAME_MAX];
554 ISAM_P isamc_p, isamc_p2;
557 strcpy(this_name, hci->cur_name);
558 assert(hci->cur_name[0]);
560 if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
562 memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
564 isamc_merge(hi->reg->isamc, &isamc_p2, isamc_i);
568 if (!dict_delete(hi->reg->dict, this_name))
574 if (isamc_p2 != isamc_p)
575 dict_insert(hi->reg->dict, this_name,
576 sizeof(ISAM_P), &isamc_p2);
582 isamc_merge(hi->reg->isamc, &isamc_p, isamc_i);
585 dict_insert(hi->reg->dict, this_name,
586 sizeof(ISAM_P), &isamc_p);
593 int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi)
595 ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
597 isamc_i->clientData = hci;
598 isamc_i->read_item = heap_cread_item2;
602 char this_name[INP_NAME_MAX];
603 ISAM_P isamc_p, isamc_p2;
606 strcpy(this_name, hci->cur_name);
607 assert(hci->cur_name[0]);
612 zebra_log_dict_entry(hi->zh, hci->cur_name);
614 if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
616 memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
618 isamb_merge(hi->reg->isamb, &isamc_p2, isamc_i);
622 if (!dict_delete(hi->reg->dict, this_name))
628 if (isamc_p2 != isamc_p)
629 dict_insert(hi->reg->dict, this_name,
630 sizeof(ISAM_P), &isamc_p2);
636 isamb_merge(hi->reg->isamb, &isamc_p, isamc_i);
639 dict_insert(hi->reg->dict, this_name,
640 sizeof(ISAM_P), &isamc_p);
647 int heap_inps(struct heap_cread_info *hci, struct heap_info *hi)
649 ISAMS_I isams_i = (ISAMS_I) xmalloc(sizeof(*isams_i));
651 isams_i->clientData = hci;
652 isams_i->read_item = heap_cread_item;
656 char this_name[INP_NAME_MAX];
660 strcpy(this_name, hci->cur_name);
661 assert(hci->cur_name[0]);
663 if (!(dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
665 isams_p = isams_merge(hi->reg->isams, isams_i);
667 dict_insert(hi->reg->dict, this_name, sizeof(ISAM_P), &isams_p);
671 yaz_log(YLOG_FATAL, "isams doesn't support this kind of update");
679 struct progressInfo {
686 void progressFunc(struct key_file *keyp, void *info)
688 struct progressInfo *p = (struct progressInfo *) info;
689 time_t now, remaining;
691 if (keyp->buf_size <= 0 || p->totalBytes <= 0)
695 if (now >= p->lastTime+10)
698 remaining = (time_t) ((now - p->startTime)*
699 ((double) p->totalBytes/p->totalOffset - 1.0));
700 if (remaining <= 130)
701 yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining",
702 (100.0*p->totalOffset) / p->totalBytes, (long) remaining);
704 yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining",
705 (100.0*p->totalOffset) / p->totalBytes, (long) remaining/60);
707 p->totalOffset += keyp->buf_size;
714 void zebra_index_merge(ZebraHandle zh)
716 struct key_file **kf = 0;
719 struct heap_info *hi;
720 struct progressInfo progressInfo;
721 int nkeys = key_block_get_no_files(zh->reg->key_block);
732 extract_get_fname_tmp (zh, fname, nkeys+1);
733 if (access(fname, R_OK) == -1)
740 kf = (struct key_file **) xmalloc((1+nkeys) * sizeof(*kf));
741 progressInfo.totalBytes = 0;
742 progressInfo.totalOffset = 0;
743 time(&progressInfo.startTime);
744 time(&progressInfo.lastTime);
745 for (i = 1; i<=nkeys; i++)
747 kf[i] = key_file_init(i, 8192, zh->res);
748 kf[i]->readHandler = progressFunc;
749 kf[i]->readInfo = &progressInfo;
750 progressInfo.totalBytes += kf[i]->length;
751 progressInfo.totalOffset += kf[i]->buf_size;
753 hi = key_heap_init_file(zh, nkeys, key_qsort_compare);
756 for (i = 1; i<=nkeys; i++)
757 if ((r = key_file_read(kf[i], rbuf)))
758 key_heap_insert(hi, rbuf, r, kf[i]);
762 struct heap_cread_info hci;
764 hci.key = (char *) xmalloc(KEY_SIZE);
765 hci.key_1 = (char *) xmalloc(KEY_SIZE);
766 hci.key_2 = (char *) xmalloc(KEY_SIZE);
768 hci.first_in_list = 1;
771 hci.more = heap_read_one(hi, hci.cur_name, hci.key);
785 for (i = 1; i<=nkeys; i++)
787 extract_get_fname_tmp (zh, rbuf, i);
790 for (i = 1; i<=nkeys; i++)
791 key_file_destroy(kf[i]);
793 if (hi->no_iterations)
794 { /* do not log if nothing happened */
795 yaz_log(YLOG_LOG, "Iterations: isam/dict "
796 ZINT_FORMAT "/" ZINT_FORMAT,
797 hi->no_iterations, hi->no_diffs);
798 yaz_log(YLOG_LOG, "Dict: inserts/updates/deletions: "
799 ZINT_FORMAT "/" ZINT_FORMAT "/" ZINT_FORMAT,
800 hi->no_insertions, hi->no_updates, hi->no_deletions);
802 key_block_destroy(&zh->reg->key_block);
803 key_heap_destroy(hi, nkeys);
808 * c-file-style: "Stroustrup"
809 * indent-tabs-mode: nil
811 * vim: shiftwidth=4 tabstop=8 expandtab