2 * Copyright (C) 1994-1995, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.7 1995-11-28 09:09:43 adam
8 * Zebra config renamed.
9 * Use setting 'recordId' to identify record now.
10 * Bug fix in recindex.c: rec_release_blocks was invokeded even
11 * though the blocks were already released.
12 * File traversal properly deletes records when needed.
14 * Revision 1.6 1995/11/25 10:24:06 adam
15 * More record fields - they are enumerated now.
16 * New options: flagStoreData flagStoreKey.
18 * Revision 1.5 1995/11/22 17:19:18 adam
19 * Record management uses the bfile system.
21 * Revision 1.4 1995/11/20 16:59:46 adam
22 * New update method: the 'old' keys are saved for each records.
24 * Revision 1.3 1995/11/16 15:34:55 adam
25 * Uses new record management system in both indexer and server.
27 * Revision 1.2 1995/11/15 19:13:08 adam
28 * Work on record management.
30 * Revision 1.1 1995/11/15 14:46:20 adam
31 * Started work on better record management system.
49 #define REC_BLOCK_TYPES 2
50 #define REC_HEAD_MAGIC "recindx"
59 char *data_fname[REC_BLOCK_TYPES];
60 BFile data_BFile[REC_BLOCK_TYPES];
65 struct record_cache_entry *record_cache;
72 int block_size[REC_BLOCK_TYPES];
73 int block_free[REC_BLOCK_TYPES];
74 int block_last[REC_BLOCK_TYPES];
75 int block_used[REC_BLOCK_TYPES];
76 int block_move[REC_BLOCK_TYPES];
85 enum recordCacheFlag { recordFlagNop, recordFlagWrite, recordFlagNew,
88 struct record_cache_entry {
90 enum recordCacheFlag flag;
93 struct record_index_entry {
106 static void rec_write_head (Records p)
111 assert (p->index_BFile);
113 r = bf_write (p->index_BFile, 0, 0, sizeof(p->head), &p->head);
116 logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname);
121 static void rec_tmp_expand (Records p, int size, int dst_type)
123 if (p->tmp_size < size + 256 ||
124 p->tmp_size < p->head.block_size[dst_type]*2)
127 p->tmp_size = size + p->head.block_size[dst_type]*2 +
129 if (!(p->tmp_buf = malloc (p->tmp_size)))
131 logf (LOG_FATAL|LOG_ERRNO, "malloc");
137 static int read_indx (Records p, int sysno, void *buf, int itemsize,
141 int pos = (sysno-1)*itemsize;
143 r = bf_read (p->index_BFile, 1+pos/128, pos%128, itemsize, buf);
144 if (r != 1 && !ignoreError)
146 logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld",
147 p->index_fname, (long) pos);
154 static void write_indx (Records p, int sysno, void *buf, int itemsize)
156 int pos = (sysno-1)*itemsize;
158 bf_write (p->index_BFile, 1+pos/128, pos%128, itemsize, buf);
161 static void rec_release_blocks (Records p, int sysno)
163 struct record_index_entry entry;
164 int freeblock, freenext;
167 if (read_indx (p, sysno, &entry, sizeof(entry), 1) != 1)
169 freeblock = entry.u.used.next;
170 assert (freeblock > 0);
171 dst_type = freeblock & 7;
172 assert (dst_type < REC_BLOCK_TYPES);
173 freeblock = freeblock / 8;
176 if (bf_read (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext),
179 logf (LOG_FATAL|LOG_ERRNO, "read in rec_del_single");
182 if (bf_write (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext),
183 &p->head.block_free[dst_type]))
185 logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single");
188 p->head.block_free[dst_type] = freeblock;
189 freeblock = freenext;
190 p->head.block_used[dst_type]--;
194 static void rec_delete_single (Records p, Record rec)
196 struct record_index_entry entry;
198 rec_release_blocks (p, rec->sysno);
200 entry.u.free.next = p->head.index_free;
201 p->head.index_free = rec->sysno;
202 write_indx (p, rec->sysno, &entry, sizeof(entry));
206 static void rec_write_single (Records p, Record rec)
212 int block_prev = -1, block_free;
213 struct record_index_entry entry;
215 for (i = 0; i < REC_NO_INFO; i++)
217 size += sizeof(*rec->size);
219 size += sizeof(*rec->size) + rec->size[i];
221 for (i = 1; i<REC_BLOCK_TYPES; i++)
222 if (size >= p->head.block_move[i])
225 rec_tmp_expand (p, size, dst_type);
227 cptr = p->tmp_buf + sizeof(int); /* a hack! */
228 for (i = 0; i < REC_NO_INFO; i++)
230 memcpy (cptr, &rec->size[i], sizeof(*rec->size));
231 cptr += sizeof(*rec->size);
234 memcpy (cptr, rec->info[i], rec->size[i]);
235 cptr += rec->size[i];
239 while (no_written < size)
241 block_free = p->head.block_free[dst_type];
244 if (bf_read (p->data_BFile[dst_type],
245 block_free, 0, sizeof(*p->head.block_free),
246 &p->head.block_free[dst_type]) != 1)
248 logf (LOG_FATAL|LOG_ERRNO, "read in %s at free block %d",
249 p->data_fname[dst_type], block_free);
253 block_free = p->head.block_last[dst_type]++;
254 if (block_prev == -1)
256 entry.u.used.next = block_free*8 + dst_type;
257 entry.u.used.size = size;
259 write_indx (p, rec->sysno, &entry, sizeof(entry));
263 memcpy (cptr, &block_free, sizeof(int));
264 logf (LOG_LOG, "writing block %d (1)", block_prev);
265 bf_write (p->data_BFile[dst_type], block_prev, 0, 0, cptr);
266 cptr = p->tmp_buf + no_written;
268 block_prev = block_free;
269 no_written += p->head.block_size[dst_type] - sizeof(int);
270 p->head.block_used[dst_type]++;
272 assert (block_prev != -1);
274 memcpy (cptr, &block_free, sizeof(int));
275 logf (LOG_LOG, "writing block %d (2) dst=%d", block_prev, dst_type);
276 bf_write (p->data_BFile[dst_type], block_prev, 0,
277 sizeof(int) + (p->tmp_buf+size) - cptr, cptr);
280 static void rec_update_single (Records p, Record rec)
282 rec_release_blocks (p, rec->sysno);
283 rec_write_single (p, rec);
286 Records rec_open (int rw)
291 if (!(p = malloc (sizeof(*p))))
293 logf (LOG_FATAL|LOG_ERRNO, "malloc");
298 p->tmp_buf = malloc (p->tmp_size);
301 logf (LOG_FATAL|LOG_ERRNO, "malloc");
304 p->index_fname = "recindex";
305 p->index_BFile = bf_open (p->index_fname, 128, rw);
306 if (p->index_BFile == NULL)
308 logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname);
311 r = bf_read (p->index_BFile, 0, 0, 0, p->tmp_buf);
315 memcpy (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
316 p->head.index_free = 0;
317 p->head.index_last = 1;
318 p->head.no_records = 0;
319 for (i = 0; i<REC_BLOCK_TYPES; i++)
321 p->head.block_free[i] = 0;
322 p->head.block_last[i] = 1;
323 p->head.block_used[i] = 0;
325 p->head.block_size[0] = 128;
326 p->head.block_move[0] = 0;
327 for (i = 1; i<REC_BLOCK_TYPES; i++)
329 p->head.block_size[i] = p->head.block_size[i-1] * 4;
330 p->head.block_move[i] = p->head.block_size[i] * 3;
336 memcpy (&p->head, p->tmp_buf, sizeof(p->head));
337 if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
339 logf (LOG_FATAL, "read %s. bad header", p->index_fname);
344 for (i = 0; i<REC_BLOCK_TYPES; i++)
347 sprintf (str, "recdata%c", i + 'A');
348 p->data_fname[i] = malloc (strlen(str)+1);
349 strcpy (p->data_fname[i], str);
350 p->data_BFile[i] = NULL;
352 for (i = 0; i<REC_BLOCK_TYPES; i++)
354 if (!(p->data_BFile[i] = bf_open (p->data_fname[i],
355 p->head.block_size[i],
358 logf (LOG_FATAL|LOG_ERRNO, "bf_open %s", p->data_fname[i]);
364 if (!(p->record_cache = malloc (sizeof(*p->record_cache)*p->cache_max)))
366 logf (LOG_FATAL|LOG_ERRNO, "malloc");
372 static void rec_cache_flush (Records p)
375 for (i = 0; i<p->cache_cur; i++)
377 struct record_cache_entry *e = p->record_cache + i;
383 rec_write_single (p, e->rec);
385 case recordFlagWrite:
386 rec_update_single (p, e->rec);
388 case recordFlagDelete:
389 rec_delete_single (p, e->rec);
397 static Record *rec_cache_lookup (Records p, int sysno,
398 enum recordCacheFlag flag)
401 for (i = 0; i<p->cache_cur; i++)
403 struct record_cache_entry *e = p->record_cache + i;
404 if (e->rec->sysno == sysno)
406 if (flag != recordFlagNop && e->flag == recordFlagNop)
414 static void rec_cache_insert (Records p, Record rec, enum recordCacheFlag flag)
416 struct record_cache_entry *e;
418 if (p->cache_cur == p->cache_max)
420 assert (p->cache_cur < p->cache_max);
422 e = p->record_cache + (p->cache_cur)++;
424 e->rec = rec_cp (rec);
427 void rec_close (Records *pp)
435 free (p->record_cache);
441 bf_close (p->index_BFile);
443 for (i = 0; i<REC_BLOCK_TYPES; i++)
445 if (p->data_BFile[i])
446 bf_close (p->data_BFile[i]);
447 free (p->data_fname[i]);
455 Record rec_get (Records p, int sysno)
459 struct record_index_entry entry;
460 int freeblock, dst_type;
466 if ((recp = rec_cache_lookup (p, sysno, recordFlagNop)))
467 return rec_cp (*recp);
469 read_indx (p, sysno, &entry, sizeof(entry), 0);
471 dst_type = entry.u.used.next & 7;
472 assert (dst_type < REC_BLOCK_TYPES);
473 freeblock = entry.u.used.next / 8;
475 assert (freeblock > 0);
477 if (!(rec = malloc (sizeof(*rec))))
479 logf (LOG_FATAL|LOG_ERRNO, "malloc");
482 rec_tmp_expand (p, entry.u.used.size, dst_type);
485 bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr);
486 memcpy (&freeblock, cptr, sizeof(freeblock));
492 cptr += p->head.block_size[dst_type] - sizeof(freeblock);
494 memcpy (&tmp, cptr, sizeof(tmp));
495 bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr);
496 memcpy (&freeblock, cptr, sizeof(freeblock));
497 memcpy (cptr, &tmp, sizeof(tmp));
501 nptr = p->tmp_buf + sizeof(freeblock);
502 for (i = 0; i < REC_NO_INFO; i++)
504 memcpy (&rec->size[i], nptr, sizeof(*rec->size));
505 nptr += sizeof(*rec->size);
508 rec->info[i] = malloc (rec->size[i]);
509 memcpy (rec->info[i], nptr, rec->size[i]);
510 nptr += rec->size[i];
515 rec_cache_insert (p, rec, recordFlagNop);
519 Record rec_new (Records p)
525 if (!(rec = malloc (sizeof(*rec))))
527 logf (LOG_FATAL|LOG_ERRNO, "malloc");
530 if (p->head.index_free == 0)
531 sysno = (p->head.index_last)++;
534 struct record_index_entry entry;
536 read_indx (p, p->head.index_free, &entry, sizeof(entry), 0);
537 sysno = p->head.index_free;
538 p->head.index_free = entry.u.free.next;
540 (p->head.no_records)++;
542 for (i = 0; i < REC_NO_INFO; i++)
547 rec_cache_insert (p, rec, recordFlagNew);
551 void rec_del (Records p, Record *recpp)
555 if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagDelete)))
562 rec_cache_insert (p, *recpp, recordFlagDelete);
568 void rec_put (Records p, Record *recpp)
572 if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagWrite)))
579 rec_cache_insert (p, *recpp, recordFlagWrite);
585 void rec_rm (Record *recpp)
588 for (i = 0; i < REC_NO_INFO; i++)
589 free ((*recpp)->info[i]);
594 Record rec_cp (Record rec)
599 if (!(n = malloc (sizeof(*n))))
601 logf (LOG_FATAL|LOG_ERRNO, "malloc");
604 n->sysno = rec->sysno;
605 for (i = 0; i < REC_NO_INFO; i++)
613 n->size[i] = rec->size[i];
614 if (!(n->info[i] = malloc (rec->size[i])))
616 logf (LOG_FATAL|LOG_ERRNO, "malloc. rec_cp");
619 memcpy (n->info[i], rec->info[i], rec->size[i]);
624 /* no BF --------------------------------------------------- */
627 struct records_info {
633 struct records_head {
647 struct record_cache_entry *record_cache;
650 struct record_cache_entry {
655 struct record_index_entry {
667 #define REC_HEAD_MAGIC "rechead"
669 static void rec_write_head (Records p)
674 assert (p->index_fd != -1);
675 if (lseek (p->index_fd, (off_t) 0, SEEK_SET) == -1)
677 logf (LOG_FATAL|LOG_ERRNO, "lseek to 0 in %s", p->index_fname);
680 r = write (p->index_fd, &p->head, sizeof(p->head));
684 logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname);
686 case sizeof(p->head):
689 logf (LOG_FATAL, "write head of %s. wrote %d", p->index_fname, r);
694 Records rec_open (int rw)
699 if (!(p = malloc (sizeof(*p))))
701 logf (LOG_FATAL|LOG_ERRNO, "malloc");
707 p->data_fname = "recdata";
709 p->index_fname = "recindex";
710 p->index_fd = open (p->index_fname,
711 rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666);
712 if (p->index_fd == -1)
714 logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname);
717 r = read (p->index_fd, &p->head, sizeof(p->head));
721 logf (LOG_FATAL|LOG_ERRNO, "read %s", p->index_fname);
724 memcpy (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
725 p->head.index_free = 0;
726 p->head.index_last = 1;
727 p->head.no_records = 0;
728 p->head.data_size = 0;
729 p->head.data_slack = 0;
730 p->head.data_used = 0;
734 case sizeof(p->head):
735 if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
737 logf (LOG_FATAL, "read %s. bad header", p->index_fname);
742 logf (LOG_FATAL, "read head of %s. expected %d. got %d",
743 p->index_fname, sizeof(p->head), r);
746 p->data_fd = open (p->data_fname,
747 rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666);
748 if (p->data_fd == -1)
750 logf (LOG_FATAL|LOG_ERRNO, "open %s", p->data_fname);
755 if (!(p->record_cache = malloc (sizeof(*p->record_cache)*p->cache_max)))
757 logf (LOG_FATAL|LOG_ERRNO, "malloc");
763 static void read_indx (Records p, int sysno, void *buf, int itemsize)
766 off_t pos = (sysno-1)*itemsize + sizeof(p->head);
768 if (lseek (p->index_fd, pos, SEEK_SET) == (pos) -1)
770 logf (LOG_FATAL|LOG_ERRNO, "seek in %s to pos %ld",
771 p->index_fname, (long) pos);
774 r = read (p->index_fd, buf, itemsize);
778 logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld",
779 p->index_fname, (long) pos);
781 logf (LOG_FATAL, "read in %s at pos %ld",
782 p->index_fname, (long) pos);
787 static void rec_write_single (Records p, Record rec)
789 struct record_index_entry entry;
790 int r, i, size = 0, got;
792 off_t pos = (rec->sysno-1)*sizeof(entry) + sizeof(p->head);
794 for (i = 0; i < REC_NO_INFO; i++)
796 size += sizeof(*rec->size);
798 size += sizeof(*rec->size) + rec->size[i];
800 entry.u.used.offset = p->head.data_size;
801 entry.u.used.size = size;
802 p->head.data_size += size;
803 p->head.data_used += size;
805 if (lseek (p->index_fd, pos, SEEK_SET) == (pos) -1)
807 logf (LOG_FATAL|LOG_ERRNO, "seek in %s to pos %ld",
808 p->index_fname, (long) pos);
811 r = write (p->index_fd, &entry, sizeof(entry));
812 if (r != sizeof(entry))
815 logf (LOG_FATAL|LOG_ERRNO, "write of %s at pos %ld",
816 p->index_fname, (long) pos);
818 logf (LOG_FATAL, "write of %s at pos %ld",
819 p->index_fname, (long) pos);
822 if (lseek (p->data_fd, entry.u.used.offset, SEEK_SET) == -1)
824 logf (LOG_FATAL|LOG_ERRNO, "lseek in %s to pos %ld",
825 p->data_fname, entry.u.used.offset);
828 if (p->tmp_size < entry.u.used.size)
831 p->tmp_size = entry.u.used.size + 16384;
832 if (!(p->tmp_buf = malloc (p->tmp_size)))
834 logf (LOG_FATAL|LOG_ERRNO, "malloc");
839 for (i = 0; i < REC_NO_INFO; i++)
841 memcpy (cptr, &rec->size[i], sizeof(*rec->size));
842 cptr += sizeof(*rec->size);
845 memcpy (cptr, rec->info[i], rec->size[i]);
846 cptr += rec->size[i];
849 for (got = 0; got < entry.u.used.size; got += r)
851 r = write (p->data_fd, p->tmp_buf + got, entry.u.used.size - got);
854 logf (LOG_FATAL|LOG_ERRNO, "write of %s", p->data_fname);
860 static void rec_cache_flush (Records p)
863 for (i = 0; i<p->cache_cur; i++)
865 struct record_cache_entry *e = p->record_cache + i;
867 rec_write_single (p, e->rec);
873 static Record *rec_cache_lookup (Records p, int sysno, int dirty)
876 for (i = 0; i<p->cache_cur; i++)
878 struct record_cache_entry *e = p->record_cache + i;
879 if (e->rec->sysno == sysno)
889 static void rec_cache_insert (Records p, Record rec, int dirty)
891 struct record_cache_entry *e;
893 if (p->cache_cur == p->cache_max)
895 assert (p->cache_cur < p->cache_max);
897 e = p->record_cache + (p->cache_cur)++;
899 e->rec = rec_cp (rec);
902 void rec_close (Records *p)
906 rec_cache_flush (*p);
907 free ((*p)->record_cache);
912 if ((*p)->index_fd != -1)
913 close ((*p)->index_fd);
915 if ((*p)->data_fd != -1)
916 close ((*p)->data_fd);
918 free ((*p)->tmp_buf);
924 Record rec_get (Records p, int sysno)
928 struct record_index_entry entry;
935 if ((recp = rec_cache_lookup (p, sysno, 0)))
936 return rec_cp (*recp);
938 read_indx (p, sysno, &entry, sizeof(entry));
940 if (!(rec = malloc (sizeof(*rec))))
942 logf (LOG_FATAL|LOG_ERRNO, "malloc");
945 if (lseek (p->data_fd, entry.u.used.offset, SEEK_SET) == -1)
947 logf (LOG_FATAL|LOG_ERRNO, "lseek in %s to pos %ld",
948 p->data_fname, entry.u.used.offset);
951 if (p->tmp_size < entry.u.used.size)
954 p->tmp_size = entry.u.used.size + 16384;
955 if (!(p->tmp_buf = malloc (p->tmp_size)))
957 logf (LOG_FATAL|LOG_ERRNO, "malloc");
961 for (got = 0; got < entry.u.used.size; got += r)
963 r = read (p->data_fd, p->tmp_buf + got, entry.u.used.size - got);
966 logf (LOG_FATAL|LOG_ERRNO, "read of %s", p->data_fname);
973 for (i = 0; i < REC_NO_INFO; i++)
975 memcpy (&rec->size[i], nptr, sizeof(*rec->size));
976 nptr += sizeof(*rec->size);
979 rec->info[i] = malloc (rec->size[i]);
980 memcpy (rec->info[i], nptr, rec->size[i]);
981 nptr += rec->size[i];
986 rec_cache_insert (p, rec, 0);
990 Record rec_new (Records p)
996 if (!(rec = malloc (sizeof(*rec))))
998 logf (LOG_FATAL|LOG_ERRNO, "malloc");
1001 if (p->head.index_free == 0)
1002 sysno = (p->head.index_last)++;
1005 struct record_index_entry entry;
1007 read_indx (p, p->head.index_free, &entry, sizeof(entry));
1008 sysno = p->head.index_free;
1009 p->head.index_free = entry.u.free.next;
1011 (p->head.no_records)++;
1013 for (i = 0; i < REC_NO_INFO; i++)
1015 rec->info[i] = NULL;
1018 rec_cache_insert (p, rec, 1);
1022 void rec_put (Records p, Record *recpp)
1026 if ((recp = rec_cache_lookup (p, (*recpp)->sysno, 1)))
1033 rec_cache_insert (p, *recpp, 1);
1039 void rec_rm (Record *recpp)
1042 for (i = 0; i < REC_NO_INFO; i++)
1043 free ((*recpp)->info[i]);
1048 Record rec_cp (Record rec)
1053 if (!(n = malloc (sizeof(*n))))
1055 logf (LOG_FATAL|LOG_ERRNO, "malloc");
1058 n->sysno = rec->sysno;
1059 for (i = 0; i < REC_NO_INFO; i++)
1067 n->size[i] = rec->size[i];
1068 if (!(n->info[i] = malloc (rec->size[i])))
1070 logf (LOG_FATAL|LOG_ERRNO, "malloc. rec_cp");
1073 memcpy (n->info[i], rec->info[i], rec->size[i]);
1078 void rec_del (Records p, Record *recpp)
1086 char *rec_strdup (const char *s, size_t *len)
1099 logf (LOG_FATAL|LOG_ERRNO, "malloc");