2 * Copyright (C) 1994-1995, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.5 1995-11-22 17:19:18 adam
8 * Record management uses the bfile system.
10 * Revision 1.4 1995/11/20 16:59:46 adam
11 * New update method: the 'old' keys are saved for each records.
13 * Revision 1.3 1995/11/16 15:34:55 adam
14 * Uses new record management system in both indexer and server.
16 * Revision 1.2 1995/11/15 19:13:08 adam
17 * Work on record management.
19 * Revision 1.1 1995/11/15 14:46:20 adam
20 * Started work on better record management system.
38 #define REC_BLOCK_TYPES 2
39 #define REC_HEAD_MAGIC "recindx"
48 char *data_fname[REC_BLOCK_TYPES];
49 BFile data_BFile[REC_BLOCK_TYPES];
54 struct record_cache_entry *record_cache;
61 int block_size[REC_BLOCK_TYPES];
62 int block_free[REC_BLOCK_TYPES];
63 int block_last[REC_BLOCK_TYPES];
64 int block_used[REC_BLOCK_TYPES];
65 int block_move[REC_BLOCK_TYPES];
74 enum recordCacheFlag { recordFlagNop, recordFlagWrite, recordFlagDelete };
76 struct record_cache_entry {
78 enum recordCacheFlag flag;
81 struct record_index_entry {
94 static void rec_write_head (Records p)
99 assert (p->index_BFile);
101 r = bf_write (p->index_BFile, 0, 0, sizeof(p->head), &p->head);
104 logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname);
109 static void rec_tmp_expand (Records p, int size, int dst_type)
111 if (p->tmp_size < size + 256 ||
112 p->tmp_size < p->head.block_size[dst_type]*2)
115 p->tmp_size = size + p->head.block_size[dst_type]*2 +
117 if (!(p->tmp_buf = malloc (p->tmp_size)))
119 logf (LOG_FATAL|LOG_ERRNO, "malloc");
125 static int read_indx (Records p, int sysno, void *buf, int itemsize,
129 int pos = (sysno-1)*itemsize;
131 r = bf_read (p->index_BFile, 1+pos/128, pos%128, itemsize, buf);
132 if (r != 1 && !ignoreError)
134 logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld",
135 p->index_fname, (long) pos);
142 static void write_indx (Records p, int sysno, void *buf, int itemsize)
144 int pos = (sysno-1)*itemsize;
146 bf_write (p->index_BFile, 1+pos/128, pos%128, itemsize, buf);
149 static void rec_release_blocks (Records p, int sysno)
151 struct record_index_entry entry;
152 int freeblock, freenext;
155 if (read_indx (p, sysno, &entry, sizeof(entry), 1) != 1)
157 freeblock = entry.u.used.next;
158 assert (freeblock > 0);
159 dst_type = freeblock & 7;
160 freeblock = freeblock / 8;
163 if (bf_read (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext),
166 logf (LOG_FATAL|LOG_ERRNO, "read in rec_del_single");
169 if (bf_write (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext),
170 &p->head.block_free[dst_type]))
172 logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single");
175 p->head.block_free[dst_type] = freeblock;
176 freeblock = freenext;
177 p->head.block_used[dst_type]--;
181 static void rec_delete_single (Records p, Record rec)
183 struct record_index_entry entry;
185 rec_release_blocks (p, rec->sysno);
187 entry.u.free.next = p->head.index_free;
188 p->head.index_free = rec->sysno;
189 write_indx (p, rec->sysno, &entry, sizeof(entry));
192 static void rec_write_single (Records p, Record rec)
198 int block_prev = -1, block_free;
199 struct record_index_entry entry;
201 rec_release_blocks (p, rec->sysno);
203 for (i = 0; i < REC_NO_INFO; i++)
205 size += sizeof(*rec->size);
207 size += sizeof(*rec->size) + rec->size[i];
209 for (i = 1; i<REC_BLOCK_TYPES; i++)
210 if (size >= p->head.block_move[i])
213 rec_tmp_expand (p, size, dst_type);
215 cptr = p->tmp_buf + sizeof(int); /* a hack! */
216 for (i = 0; i < REC_NO_INFO; i++)
218 memcpy (cptr, &rec->size[i], sizeof(*rec->size));
219 cptr += sizeof(*rec->size);
222 memcpy (cptr, rec->info[i], rec->size[i]);
223 cptr += rec->size[i];
227 while (no_written < size)
229 block_free = p->head.block_free[dst_type];
232 if (bf_read (p->data_BFile[dst_type],
233 block_free, 0, sizeof(*p->head.block_free),
234 &p->head.block_free[dst_type]) != 1)
236 logf (LOG_FATAL|LOG_ERRNO, "read in %s at free block %d",
237 p->data_fname[dst_type], block_free);
241 block_free = p->head.block_last[dst_type]++;
242 if (block_prev == -1)
244 entry.u.used.next = block_free*8 + dst_type;
245 entry.u.used.size = size;
247 write_indx (p, rec->sysno, &entry, sizeof(entry));
251 memcpy (cptr, &block_free, sizeof(int));
252 bf_write (p->data_BFile[dst_type], block_prev, 0, 0, cptr);
253 cptr = p->tmp_buf + no_written;
255 block_prev = block_free;
256 no_written += p->head.block_size[dst_type] - sizeof(int);
257 p->head.block_used[dst_type]++;
259 assert (block_prev != -1);
261 memcpy (cptr, &block_free, sizeof(int));
262 bf_write (p->data_BFile[dst_type], block_prev, 0,
263 sizeof(int) + (p->tmp_buf+size) - cptr, cptr);
267 Records rec_open (int rw)
272 if (!(p = malloc (sizeof(*p))))
274 logf (LOG_FATAL|LOG_ERRNO, "malloc");
279 p->tmp_buf = malloc (p->tmp_size);
282 logf (LOG_FATAL|LOG_ERRNO, "malloc");
285 p->index_fname = "recindex";
286 p->index_BFile = bf_open (p->index_fname, 128, rw);
287 if (p->index_BFile == NULL)
289 logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname);
292 r = bf_read (p->index_BFile, 0, 0, 0, p->tmp_buf);
296 memcpy (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
297 p->head.index_free = 0;
298 p->head.index_last = 1;
299 p->head.no_records = 0;
300 for (i = 0; i<REC_BLOCK_TYPES; i++)
302 p->head.block_free[i] = 0;
303 p->head.block_last[i] = 1;
304 p->head.block_used[i] = 0;
306 p->head.block_size[0] = 128;
307 p->head.block_move[0] = 0;
308 for (i = 1; i<REC_BLOCK_TYPES; i++)
310 p->head.block_size[i] = p->head.block_size[i-1] * 4;
311 p->head.block_move[i] = p->head.block_size[i] * 3;
317 memcpy (&p->head, p->tmp_buf, sizeof(p->head));
318 if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
320 logf (LOG_FATAL, "read %s. bad header", p->index_fname);
325 for (i = 0; i<REC_BLOCK_TYPES; i++)
328 sprintf (str, "recdata%d", i);
329 p->data_fname[i] = malloc (strlen(str)+1);
330 strcpy (p->data_fname[i], str);
331 p->data_BFile[i] = NULL;
333 for (i = 0; i<REC_BLOCK_TYPES; i++)
335 if (!(p->data_BFile[i] = bf_open (p->data_fname[i],
336 p->head.block_size[i],
339 logf (LOG_FATAL|LOG_ERRNO, "bf_open %s", p->data_fname[i]);
345 if (!(p->record_cache = malloc (sizeof(*p->record_cache)*p->cache_max)))
347 logf (LOG_FATAL|LOG_ERRNO, "malloc");
353 static void rec_cache_flush (Records p)
356 for (i = 0; i<p->cache_cur; i++)
358 struct record_cache_entry *e = p->record_cache + i;
363 case recordFlagWrite:
364 rec_write_single (p, e->rec);
366 case recordFlagDelete:
367 rec_delete_single (p, e->rec);
375 static Record *rec_cache_lookup (Records p, int sysno,
376 enum recordCacheFlag flag)
379 for (i = 0; i<p->cache_cur; i++)
381 struct record_cache_entry *e = p->record_cache + i;
382 if (e->rec->sysno == sysno)
384 if (flag != recordFlagNop)
392 static void rec_cache_insert (Records p, Record rec, enum recordCacheFlag flag)
394 struct record_cache_entry *e;
396 if (p->cache_cur == p->cache_max)
398 assert (p->cache_cur < p->cache_max);
400 e = p->record_cache + (p->cache_cur)++;
402 e->rec = rec_cp (rec);
405 void rec_close (Records *pp)
413 free (p->record_cache);
419 bf_close (p->index_BFile);
421 for (i = 0; i<REC_BLOCK_TYPES; i++)
423 if (p->data_BFile[i])
424 bf_close (p->data_BFile[i]);
425 free (p->data_fname[i]);
433 Record rec_get (Records p, int sysno)
437 struct record_index_entry entry;
438 int freeblock, dst_type;
444 if ((recp = rec_cache_lookup (p, sysno, recordFlagNop)))
445 return rec_cp (*recp);
447 read_indx (p, sysno, &entry, sizeof(entry), 0);
449 dst_type = entry.u.used.next & 7;
450 freeblock = entry.u.used.next / 8;
452 assert (freeblock > 0);
454 if (!(rec = malloc (sizeof(*rec))))
456 logf (LOG_FATAL|LOG_ERRNO, "malloc");
459 rec_tmp_expand (p, entry.u.used.size, dst_type);
462 bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr);
463 memcpy (&freeblock, cptr, sizeof(freeblock));
469 cptr += p->head.block_size[dst_type] - sizeof(freeblock);
471 memcpy (&tmp, cptr, sizeof(tmp));
472 bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr);
473 memcpy (&freeblock, cptr, sizeof(freeblock));
474 memcpy (cptr, &tmp, sizeof(tmp));
478 nptr = p->tmp_buf + sizeof(freeblock);
479 for (i = 0; i < REC_NO_INFO; i++)
481 memcpy (&rec->size[i], nptr, sizeof(*rec->size));
482 nptr += sizeof(*rec->size);
485 rec->info[i] = malloc (rec->size[i]);
486 memcpy (rec->info[i], nptr, rec->size[i]);
487 nptr += rec->size[i];
492 rec_cache_insert (p, rec, recordFlagNop);
496 Record rec_new (Records p)
502 if (!(rec = malloc (sizeof(*rec))))
504 logf (LOG_FATAL|LOG_ERRNO, "malloc");
507 if (p->head.index_free == 0)
508 sysno = (p->head.index_last)++;
511 struct record_index_entry entry;
513 read_indx (p, p->head.index_free, &entry, sizeof(entry), 0);
514 sysno = p->head.index_free;
515 p->head.index_free = entry.u.free.next;
517 (p->head.no_records)++;
519 for (i = 0; i < REC_NO_INFO; i++)
524 rec_cache_insert (p, rec, recordFlagWrite);
528 void rec_del (Records p, Record *recpp)
532 if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagDelete)))
539 rec_cache_insert (p, *recpp, recordFlagDelete);
545 void rec_put (Records p, Record *recpp)
549 if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagWrite)))
556 rec_cache_insert (p, *recpp, recordFlagWrite);
562 void rec_rm (Record *recpp)
565 for (i = 0; i < REC_NO_INFO; i++)
566 free ((*recpp)->info[i]);
571 Record rec_cp (Record rec)
576 if (!(n = malloc (sizeof(*n))))
578 logf (LOG_FATAL|LOG_ERRNO, "malloc");
581 n->sysno = rec->sysno;
582 for (i = 0; i < REC_NO_INFO; i++)
590 n->size[i] = rec->size[i];
591 if (!(n->info[i] = malloc (rec->size[i])))
593 logf (LOG_FATAL|LOG_ERRNO, "malloc. rec_cp");
596 memcpy (n->info[i], rec->info[i], rec->size[i]);
601 /* no BF --------------------------------------------------- */
604 struct records_info {
610 struct records_head {
624 struct record_cache_entry *record_cache;
627 struct record_cache_entry {
632 struct record_index_entry {
644 #define REC_HEAD_MAGIC "rechead"
646 static void rec_write_head (Records p)
651 assert (p->index_fd != -1);
652 if (lseek (p->index_fd, (off_t) 0, SEEK_SET) == -1)
654 logf (LOG_FATAL|LOG_ERRNO, "lseek to 0 in %s", p->index_fname);
657 r = write (p->index_fd, &p->head, sizeof(p->head));
661 logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname);
663 case sizeof(p->head):
666 logf (LOG_FATAL, "write head of %s. wrote %d", p->index_fname, r);
671 Records rec_open (int rw)
676 if (!(p = malloc (sizeof(*p))))
678 logf (LOG_FATAL|LOG_ERRNO, "malloc");
684 p->data_fname = "recdata";
686 p->index_fname = "recindex";
687 p->index_fd = open (p->index_fname,
688 rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666);
689 if (p->index_fd == -1)
691 logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname);
694 r = read (p->index_fd, &p->head, sizeof(p->head));
698 logf (LOG_FATAL|LOG_ERRNO, "read %s", p->index_fname);
701 memcpy (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
702 p->head.index_free = 0;
703 p->head.index_last = 1;
704 p->head.no_records = 0;
705 p->head.data_size = 0;
706 p->head.data_slack = 0;
707 p->head.data_used = 0;
711 case sizeof(p->head):
712 if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
714 logf (LOG_FATAL, "read %s. bad header", p->index_fname);
719 logf (LOG_FATAL, "read head of %s. expected %d. got %d",
720 p->index_fname, sizeof(p->head), r);
723 p->data_fd = open (p->data_fname,
724 rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666);
725 if (p->data_fd == -1)
727 logf (LOG_FATAL|LOG_ERRNO, "open %s", p->data_fname);
732 if (!(p->record_cache = malloc (sizeof(*p->record_cache)*p->cache_max)))
734 logf (LOG_FATAL|LOG_ERRNO, "malloc");
740 static void read_indx (Records p, int sysno, void *buf, int itemsize)
743 off_t pos = (sysno-1)*itemsize + sizeof(p->head);
745 if (lseek (p->index_fd, pos, SEEK_SET) == (pos) -1)
747 logf (LOG_FATAL|LOG_ERRNO, "seek in %s to pos %ld",
748 p->index_fname, (long) pos);
751 r = read (p->index_fd, buf, itemsize);
755 logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld",
756 p->index_fname, (long) pos);
758 logf (LOG_FATAL, "read in %s at pos %ld",
759 p->index_fname, (long) pos);
764 static void rec_write_single (Records p, Record rec)
766 struct record_index_entry entry;
767 int r, i, size = 0, got;
769 off_t pos = (rec->sysno-1)*sizeof(entry) + sizeof(p->head);
771 for (i = 0; i < REC_NO_INFO; i++)
773 size += sizeof(*rec->size);
775 size += sizeof(*rec->size) + rec->size[i];
777 entry.u.used.offset = p->head.data_size;
778 entry.u.used.size = size;
779 p->head.data_size += size;
780 p->head.data_used += size;
782 if (lseek (p->index_fd, pos, SEEK_SET) == (pos) -1)
784 logf (LOG_FATAL|LOG_ERRNO, "seek in %s to pos %ld",
785 p->index_fname, (long) pos);
788 r = write (p->index_fd, &entry, sizeof(entry));
789 if (r != sizeof(entry))
792 logf (LOG_FATAL|LOG_ERRNO, "write of %s at pos %ld",
793 p->index_fname, (long) pos);
795 logf (LOG_FATAL, "write of %s at pos %ld",
796 p->index_fname, (long) pos);
799 if (lseek (p->data_fd, entry.u.used.offset, SEEK_SET) == -1)
801 logf (LOG_FATAL|LOG_ERRNO, "lseek in %s to pos %ld",
802 p->data_fname, entry.u.used.offset);
805 if (p->tmp_size < entry.u.used.size)
808 p->tmp_size = entry.u.used.size + 16384;
809 if (!(p->tmp_buf = malloc (p->tmp_size)))
811 logf (LOG_FATAL|LOG_ERRNO, "malloc");
816 for (i = 0; i < REC_NO_INFO; i++)
818 memcpy (cptr, &rec->size[i], sizeof(*rec->size));
819 cptr += sizeof(*rec->size);
822 memcpy (cptr, rec->info[i], rec->size[i]);
823 cptr += rec->size[i];
826 for (got = 0; got < entry.u.used.size; got += r)
828 r = write (p->data_fd, p->tmp_buf + got, entry.u.used.size - got);
831 logf (LOG_FATAL|LOG_ERRNO, "write of %s", p->data_fname);
837 static void rec_cache_flush (Records p)
840 for (i = 0; i<p->cache_cur; i++)
842 struct record_cache_entry *e = p->record_cache + i;
844 rec_write_single (p, e->rec);
850 static Record *rec_cache_lookup (Records p, int sysno, int dirty)
853 for (i = 0; i<p->cache_cur; i++)
855 struct record_cache_entry *e = p->record_cache + i;
856 if (e->rec->sysno == sysno)
866 static void rec_cache_insert (Records p, Record rec, int dirty)
868 struct record_cache_entry *e;
870 if (p->cache_cur == p->cache_max)
872 assert (p->cache_cur < p->cache_max);
874 e = p->record_cache + (p->cache_cur)++;
876 e->rec = rec_cp (rec);
879 void rec_close (Records *p)
883 rec_cache_flush (*p);
884 free ((*p)->record_cache);
889 if ((*p)->index_fd != -1)
890 close ((*p)->index_fd);
892 if ((*p)->data_fd != -1)
893 close ((*p)->data_fd);
895 free ((*p)->tmp_buf);
901 Record rec_get (Records p, int sysno)
905 struct record_index_entry entry;
912 if ((recp = rec_cache_lookup (p, sysno, 0)))
913 return rec_cp (*recp);
915 read_indx (p, sysno, &entry, sizeof(entry));
917 if (!(rec = malloc (sizeof(*rec))))
919 logf (LOG_FATAL|LOG_ERRNO, "malloc");
922 if (lseek (p->data_fd, entry.u.used.offset, SEEK_SET) == -1)
924 logf (LOG_FATAL|LOG_ERRNO, "lseek in %s to pos %ld",
925 p->data_fname, entry.u.used.offset);
928 if (p->tmp_size < entry.u.used.size)
931 p->tmp_size = entry.u.used.size + 16384;
932 if (!(p->tmp_buf = malloc (p->tmp_size)))
934 logf (LOG_FATAL|LOG_ERRNO, "malloc");
938 for (got = 0; got < entry.u.used.size; got += r)
940 r = read (p->data_fd, p->tmp_buf + got, entry.u.used.size - got);
943 logf (LOG_FATAL|LOG_ERRNO, "read of %s", p->data_fname);
950 for (i = 0; i < REC_NO_INFO; i++)
952 memcpy (&rec->size[i], nptr, sizeof(*rec->size));
953 nptr += sizeof(*rec->size);
956 rec->info[i] = malloc (rec->size[i]);
957 memcpy (rec->info[i], nptr, rec->size[i]);
958 nptr += rec->size[i];
963 rec_cache_insert (p, rec, 0);
967 Record rec_new (Records p)
973 if (!(rec = malloc (sizeof(*rec))))
975 logf (LOG_FATAL|LOG_ERRNO, "malloc");
978 if (p->head.index_free == 0)
979 sysno = (p->head.index_last)++;
982 struct record_index_entry entry;
984 read_indx (p, p->head.index_free, &entry, sizeof(entry));
985 sysno = p->head.index_free;
986 p->head.index_free = entry.u.free.next;
988 (p->head.no_records)++;
990 for (i = 0; i < REC_NO_INFO; i++)
995 rec_cache_insert (p, rec, 1);
999 void rec_put (Records p, Record *recpp)
1003 if ((recp = rec_cache_lookup (p, (*recpp)->sysno, 1)))
1010 rec_cache_insert (p, *recpp, 1);
1016 void rec_rm (Record *recpp)
1019 for (i = 0; i < REC_NO_INFO; i++)
1020 free ((*recpp)->info[i]);
1025 Record rec_cp (Record rec)
1030 if (!(n = malloc (sizeof(*n))))
1032 logf (LOG_FATAL|LOG_ERRNO, "malloc");
1035 n->sysno = rec->sysno;
1036 for (i = 0; i < REC_NO_INFO; i++)
1044 n->size[i] = rec->size[i];
1045 if (!(n->info[i] = malloc (rec->size[i])))
1047 logf (LOG_FATAL|LOG_ERRNO, "malloc. rec_cp");
1050 memcpy (n->info[i], rec->info[i], rec->size[i]);
1055 void rec_del (Records p, Record *recpp)
1063 char *rec_strdup (const char *s, size_t *len)
1076 logf (LOG_FATAL|LOG_ERRNO, "malloc");