2 * Copyright (C) 1994-1995, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.10 1995-12-06 12:41:22 adam
8 * New command 'stat' for the index program.
9 * Filenames can be read from stdin by specifying '-'.
10 * Bug fix/enhancement of the transformation from terms to regular
11 * expressons in the search engine.
13 * Revision 1.9 1995/10/10 12:24:39 adam
14 * Temporary sort files are compressed.
16 * Revision 1.8 1995/10/04 16:57:19 adam
17 * Key input and merge sort in one pass.
19 * Revision 1.7 1995/10/02 15:18:52 adam
20 * New member in recRetrieveCtrl: diagnostic.
22 * Revision 1.6 1995/09/29 15:51:56 adam
23 * First work on multi-way read.
25 * Revision 1.5 1995/09/29 14:01:43 adam
28 * Revision 1.4 1995/09/28 14:22:57 adam
29 * Sort uses smaller temporary files.
31 * Revision 1.3 1995/09/06 16:11:17 adam
32 * Option: only one word key per file.
34 * Revision 1.2 1995/09/04 12:33:42 adam
35 * Various cleanup. YAZ util used instead.
37 * Revision 1.1 1995/09/04 09:10:37 adam
38 * More work on index add/del/update.
39 * Merge sort implemented.
40 * Initial work on z39 server.
54 #define KEY_SIZE (1+sizeof(struct it_key))
55 #define INP_NAME_MAX 8192
56 #define INP_BUF_START 60000
57 #define INP_BUF_ADD 400000
59 static int no_diffs = 0;
60 static int no_updates = 0;
61 static int no_insertions = 0;
62 static int no_iterations = 0;
66 off_t offset; /* file offset */
67 unsigned char *buf; /* buffer block */
68 size_t buf_size; /* number of read bytes in block */
69 size_t chunk; /* number of bytes allocated */
70 size_t buf_ptr; /* current position in buffer */
71 char *prev_name; /* last word read */
72 int sysno; /* last sysno */
73 int seqno; /* last seqno */
76 void key_file_chunk_read (struct key_file *f)
80 sprintf (fname, TEMP_FNAME, f->no);
81 fd = open (fname, O_RDONLY);
84 logf (LOG_FATAL|LOG_ERRNO, "cannot open %s", fname);
87 logf (LOG_LOG, "reading chunk from %s", fname);
88 if (lseek (fd, f->offset, SEEK_SET) == -1)
90 logf (LOG_FATAL|LOG_ERRNO, "cannot seek %s", fname);
93 while (f->chunk - nr > 0)
95 r = read (fd, f->buf + nr, f->chunk - nr);
102 logf (LOG_FATAL|LOG_ERRNO, "read of %s", fname);
110 struct key_file *key_file_init (int no, int chunk)
114 f = xmalloc (sizeof(*f));
120 f->buf = xmalloc (f->chunk);
121 f->prev_name = xmalloc (256);
122 *f->prev_name = '\0';
123 key_file_chunk_read (f);
127 int key_file_getc (struct key_file *f)
129 if (f->buf_ptr < f->buf_size)
130 return f->buf[(f->buf_ptr)++];
131 if (f->buf_size < f->chunk)
133 f->offset += f->buf_size;
134 key_file_chunk_read (f);
135 if (f->buf_ptr < f->buf_size)
136 return f->buf[(f->buf_ptr)++];
141 int key_file_decode (struct key_file *f)
145 c = key_file_getc (f);
152 d = ((c&63) << 8) + (key_file_getc (f) & 0xff);
155 d = ((c&63) << 8) + (key_file_getc (f) & 0xff);
156 d = (d << 8) + (key_file_getc (f) & 0xff);
159 d = ((c&63) << 8) + (key_file_getc (f) & 0xff);
160 d = (d << 8) + (key_file_getc (f) & 0xff);
161 d = (d << 8) + (key_file_getc (f) & 0xff);
167 int key_file_read (struct key_file *f, char *key)
172 c = key_file_getc (f);
175 strcpy (key, f->prev_name);
184 while ((key[i++] = key_file_getc (f)))
186 strcpy (f->prev_name, key);
189 d = key_file_decode (f);
192 itkey.sysno = d + f->sysno;
195 f->sysno = itkey.sysno;
198 d = key_file_decode (f);
199 itkey.seqno = d + f->seqno;
200 f->seqno = itkey.seqno;
201 memcpy (key + i, &itkey, sizeof(struct it_key));
202 return i + sizeof (struct it_key);
207 struct key_file **file;
212 int (*cmp)(const void *p1, const void *p2);
215 struct heap_info *key_heap_init (int nkeys,
216 int (*cmp)(const void *p1, const void *p2))
218 struct heap_info *hi;
221 hi = xmalloc (sizeof(*hi));
222 hi->info.file = xmalloc (sizeof(*hi->info.file) * (1+nkeys));
223 hi->info.buf = xmalloc (sizeof(*hi->info.buf) * (1+nkeys));
225 hi->ptr = xmalloc (sizeof(*hi->ptr) * (1+nkeys));
227 for (i = 0; i<= nkeys; i++)
230 hi->info.buf[i] = xmalloc (512);
235 static void key_heap_swap (struct heap_info *hi, int i1, int i2)
240 hi->ptr[i1] = hi->ptr[i2];
245 static void key_heap_delete (struct heap_info *hi)
247 int cur = 1, child = 2;
249 assert (hi->heapnum > 0);
251 key_heap_swap (hi, 1, hi->heapnum);
253 while (child <= hi->heapnum) {
254 if (child < hi->heapnum &&
255 (*hi->cmp)(&hi->info.buf[hi->ptr[child]],
256 &hi->info.buf[hi->ptr[child+1]]) > 0)
258 if ((*hi->cmp)(&hi->info.buf[hi->ptr[cur]],
259 &hi->info.buf[hi->ptr[child]]) > 0)
261 key_heap_swap (hi, cur, child);
270 static void key_heap_insert (struct heap_info *hi, const char *buf, int nbytes,
275 cur = ++(hi->heapnum);
276 memcpy (hi->info.buf[hi->ptr[cur]], buf, nbytes);
277 hi->info.file[hi->ptr[cur]] = kf;
280 while (parent && (*hi->cmp)(&hi->info.buf[hi->ptr[parent]],
281 &hi->info.buf[hi->ptr[cur]]) > 0)
283 key_heap_swap (hi, cur, parent);
289 static int heap_read_one (struct heap_info *hi, char *name, char *key)
298 strcpy (name, hi->info.buf[n]);
299 kf = hi->info.file[n];
301 memcpy (key, hi->info.buf[n] + r+1, KEY_SIZE);
302 key_heap_delete (hi);
303 if ((r = key_file_read (kf, rbuf)))
304 key_heap_insert (hi, rbuf, r, kf);
309 int heap_inp (Dict dict, ISAM isam, struct heap_info *hi)
312 char next_name[INP_NAME_MAX+1];
313 char cur_name[INP_NAME_MAX+1];
314 int key_buf_size = INP_BUF_START;
320 next_key = xmalloc (KEY_SIZE);
321 key_buf = xmalloc (key_buf_size * (KEY_SIZE));
322 more = heap_read_one (hi, cur_name, key_buf);
323 while (more) /* EOF ? */
326 key_buf_ptr = KEY_SIZE;
329 if (!(more = heap_read_one (hi, next_name, next_key)))
331 if (*next_name && strcmp (next_name, cur_name))
333 memcpy (key_buf + key_buf_ptr, next_key, KEY_SIZE);
334 key_buf_ptr += KEY_SIZE;
335 if (key_buf_ptr+KEY_SIZE >= key_buf_size)
338 new_key_buf = xmalloc (key_buf_size + INP_BUF_ADD);
339 memcpy (new_key_buf, key_buf, key_buf_size);
340 key_buf_size += INP_BUF_ADD;
342 key_buf = new_key_buf;
346 nmemb = key_buf_ptr / KEY_SIZE;
347 assert (nmemb*KEY_SIZE == key_buf_ptr);
348 if ((info = dict_lookup (dict, cur_name)))
350 ISAM_P isam_p, isam_p2;
351 logf (LOG_DEBUG, "updating %s", cur_name);
353 memcpy (&isam_p, info+1, sizeof(ISAM_P));
354 isam_p2 = is_merge (isam, isam_p, nmemb, key_buf);
355 if (isam_p2 != isam_p)
356 dict_insert (dict, cur_name, sizeof(ISAM_P), &isam_p2);
361 logf (LOG_DEBUG, "inserting %s", cur_name);
363 isam_p = is_merge (isam, 0, nmemb, key_buf);
364 dict_insert (dict, cur_name, sizeof(ISAM_P), &isam_p);
366 memcpy (key_buf, next_key, KEY_SIZE);
367 strcpy (cur_name, next_name);
372 void key_input (const char *dict_fname, const char *isam_fname,
373 int nkeys, int cache)
378 struct key_file **kf;
381 struct heap_info *hi;
383 dict = dict_open (dict_fname, cache, 1);
386 logf (LOG_FATAL, "dict_open fail of `%s'", dict_fname);
389 isam = is_open (isam_fname, key_compare, 1, sizeof(struct it_key));
392 logf (LOG_FATAL, "is_open fail of `%s'", isam_fname);
396 kf = xmalloc ((1+nkeys) * sizeof(*kf));
397 for (i = 1; i<=nkeys; i++)
398 kf[i] = key_file_init (i, 32768);
400 hi = key_heap_init (nkeys, key_qsort_compare);
401 for (i = 1; i<=nkeys; i++)
402 if ((r = key_file_read (kf[i], rbuf)))
403 key_heap_insert (hi, rbuf, r, kf[i]);
404 heap_inp (dict, isam, hi);
409 logf (LOG_LOG, "Iterations . . .%7d", no_iterations);
410 logf (LOG_LOG, "Distinct words .%7d", no_diffs);
411 logf (LOG_LOG, "Updates. . . . .%7d", no_updates);
412 logf (LOG_LOG, "Insertions . . .%7d", no_insertions);