1 /* $Id: cfile.c,v 1.31 2004-11-19 10:26:53 heikki Exp $
2 Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
33 static int write_head (CFile cf)
35 int left = cf->head.hash_size * sizeof(zint);
37 const char *tab = (char*) cf->array;
41 while (left >= (int) HASH_BSIZE)
43 mf_write (cf->hash_mf, bno++, 0, 0, tab);
48 mf_write (cf->hash_mf, bno, 0, left, tab);
52 static int read_head (CFile cf)
54 int left = cf->head.hash_size * sizeof(zint);
56 char *tab = (char*) cf->array;
60 while (left >= (int) HASH_BSIZE)
62 mf_read (cf->hash_mf, bno++, 0, 0, tab);
67 mf_read (cf->hash_mf, bno, 0, left, tab);
72 CFile cf_open (MFile mf, MFile_area area, const char *fname,
73 int block_size, int wflag, int *firstp)
77 CFile cf = (CFile) xmalloc (sizeof(*cf));
81 yaz_log (YLOG_DEBUG, "cf: open %s %s", cf->rmf->name, wflag ? "rdwr" : "rd");
82 sprintf (path, "%s-b", fname);
83 if (!(cf->block_mf = mf_open (area, path, block_size, wflag)))
85 yaz_log (YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
88 sprintf (path, "%s-i", fname);
89 if (!(cf->hash_mf = mf_open (area, path, HASH_BSIZE, wflag)))
91 yaz_log (YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
95 if (!mf_read (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head) ||
100 cf->head.block_size = block_size;
101 cf->head.hash_size = 199;
102 hash_bytes = cf->head.hash_size * sizeof(zint);
103 cf->head.flat_bucket = cf->head.next_bucket = cf->head.first_bucket =
104 (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2;
105 cf->head.next_block = 1;
107 mf_write (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
108 cf->array = (zint *) xmalloc (hash_bytes);
109 for (i = 0; i<cf->head.hash_size; i++)
117 assert (cf->head.block_size == block_size);
118 assert (cf->head.hash_size > 2);
119 hash_bytes = cf->head.hash_size * sizeof(zint);
120 assert (cf->head.next_bucket > 0);
121 assert (cf->head.next_block > 0);
122 if (cf->head.state == 1)
123 cf->array = (zint *) xmalloc (hash_bytes);
128 if (cf->head.state == 1)
130 cf->parray = (struct CFile_hash_bucket **)
131 xmalloc (cf->head.hash_size * sizeof(*cf->parray));
132 for (i = 0; i<cf->head.hash_size; i++)
133 cf->parray[i] = NULL;
137 cf->bucket_lru_front = cf->bucket_lru_back = NULL;
138 cf->bucket_in_memory = 0;
139 cf->max_bucket_in_memory = 100;
141 cf->iobuf = (char *) xmalloc (cf->head.block_size);
142 memset (cf->iobuf, 0, cf->head.block_size);
145 zebra_mutex_init (&cf->mutex);
149 static int cf_hash (CFile cf, zint no)
151 return (int) (((no >> 3) % cf->head.hash_size));
154 static void release_bucket (CFile cf, struct CFile_hash_bucket *p)
157 p->lru_prev->lru_next = p->lru_next;
159 cf->bucket_lru_back = p->lru_next;
161 p->lru_next->lru_prev = p->lru_prev;
163 cf->bucket_lru_front = p->lru_prev;
165 *p->h_prev = p->h_next;
167 p->h_next->h_prev = p->h_prev;
169 --(cf->bucket_in_memory);
173 static void flush_bucket (CFile cf, int no_to_flush)
176 struct CFile_hash_bucket *p;
178 for (i = 0; i != no_to_flush; i++)
180 p = cf->bucket_lru_back;
185 mf_write (cf->hash_mf, p->ph.this_bucket, 0, 0, &p->ph);
188 release_bucket (cf, p);
192 static struct CFile_hash_bucket *alloc_bucket (CFile cf, zint block_no, int hno)
194 struct CFile_hash_bucket *p, **pp;
196 if (cf->bucket_in_memory == cf->max_bucket_in_memory)
197 flush_bucket (cf, 1);
198 assert (cf->bucket_in_memory < cf->max_bucket_in_memory);
199 ++(cf->bucket_in_memory);
200 p = (struct CFile_hash_bucket *) xmalloc (sizeof(*p));
203 p->lru_prev = cf->bucket_lru_front;
204 if (cf->bucket_lru_front)
205 cf->bucket_lru_front->lru_next = p;
207 cf->bucket_lru_back = p;
208 cf->bucket_lru_front = p;
210 pp = cf->parray + hno;
214 (*pp)->h_prev = &p->h_next;
219 static struct CFile_hash_bucket *get_bucket (CFile cf, zint block_no, int hno)
221 struct CFile_hash_bucket *p;
223 p = alloc_bucket (cf, block_no, hno);
224 if (!mf_read (cf->hash_mf, block_no, 0, 0, &p->ph))
226 yaz_log (YLOG_FATAL|YLOG_ERRNO, "read get_bucket");
229 assert (p->ph.this_bucket == block_no);
234 static struct CFile_hash_bucket *new_bucket (CFile cf, zint *block_nop, int hno)
236 struct CFile_hash_bucket *p;
240 block_no = *block_nop = cf->head.next_bucket++;
241 p = alloc_bucket (cf, block_no, hno);
243 for (i = 0; i<HASH_BUCKET; i++)
248 p->ph.next_bucket = 0;
249 p->ph.this_bucket = block_no;
254 static zint cf_lookup_flat (CFile cf, zint no)
256 zint hno = (no*sizeof(zint))/HASH_BSIZE;
257 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
260 mf_read (cf->hash_mf, hno+cf->head.next_bucket, off, sizeof(zint), &vno);
264 static zint cf_lookup_hash (CFile cf, zint no)
266 int hno = cf_hash (cf, no);
267 struct CFile_hash_bucket *hb;
271 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
273 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
274 if (hb->ph.no[i] == no)
277 return hb->ph.vno[i];
280 for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket)
282 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
284 if (hb->ph.this_bucket == block_no)
290 /* extra check ... */
291 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
293 if (hb->ph.this_bucket == block_no)
295 yaz_log (YLOG_FATAL, "Found hash bucket on other chain (1)");
298 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
299 if (hb->ph.no[i] == no)
301 yaz_log (YLOG_FATAL, "Found hash bucket on other chain (2)");
307 hb = get_bucket (cf, block_no, hno);
308 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
309 if (hb->ph.no[i] == no)
310 return hb->ph.vno[i];
315 static void cf_write_flat (CFile cf, zint no, zint vno)
317 zint hno = (no*sizeof(zint))/HASH_BSIZE;
318 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
320 hno += cf->head.next_bucket;
321 if (hno >= cf->head.flat_bucket)
322 cf->head.flat_bucket = hno+1;
324 mf_write (cf->hash_mf, hno, off, sizeof(zint), &vno);
327 static void cf_moveto_flat (CFile cf)
329 struct CFile_hash_bucket *p;
333 yaz_log (YLOG_DEBUG, "cf: Moving to flat shadow: %s", cf->rmf->name);
334 yaz_log (YLOG_DEBUG, "cf: hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT " total="
336 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
337 cf->head.next_bucket - cf->head.first_bucket);
338 assert (cf->head.state == 1);
339 flush_bucket (cf, -1);
340 assert (cf->bucket_in_memory == 0);
341 p = (struct CFile_hash_bucket *) xmalloc (sizeof(*p));
342 for (i = cf->head.first_bucket; i < cf->head.next_bucket; i++)
344 if (!mf_read (cf->hash_mf, i, 0, 0, &p->ph))
346 yaz_log (YLOG_FATAL|YLOG_ERRNO, "read bucket moveto flat");
349 for (j = 0; j < HASH_BUCKET && p->ph.vno[j]; j++)
350 cf_write_flat (cf, p->ph.no[j], p->ph.vno[j]);
361 static zint cf_lookup (CFile cf, zint no)
363 if (cf->head.state > 1)
364 return cf_lookup_flat (cf, no);
365 return cf_lookup_hash (cf, no);
368 static zint cf_new_flat (CFile cf, zint no)
370 zint vno = (cf->head.next_block)++;
372 cf_write_flat (cf, no, vno);
376 static zint cf_new_hash (CFile cf, zint no)
378 int hno = cf_hash (cf, no);
379 struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno];
380 zint *bucketpp = &cf->array[hno];
382 zint vno = (cf->head.next_block)++;
384 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
385 if (!hb->ph.vno[HASH_BUCKET-1])
386 for (i = 0; i<HASH_BUCKET; i++)
398 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
399 if (hb->ph.this_bucket == *bucketpp)
401 bucketpp = &hb->ph.next_bucket;
409 /* extra check ... */
410 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
412 if (hb->ph.this_bucket == *bucketpp)
414 yaz_log (YLOG_FATAL, "Found hash bucket on other chain");
420 hb = get_bucket (cf, *bucketpp, hno);
422 for (i = 0; i<HASH_BUCKET; i++)
430 bucketpp = &hb->ph.next_bucket;
435 hb = new_bucket (cf, bucketpp, hno);
441 zint cf_new (CFile cf, zint no)
443 if (cf->head.state > 1)
444 return cf_new_flat (cf, no);
445 if (cf->no_miss*2 > cf->no_hits)
448 assert (cf->head.state > 1);
449 return cf_new_flat (cf, no);
451 return cf_new_hash (cf, no);
455 int cf_read (CFile cf, zint no, int offset, int nbytes, void *buf)
460 zebra_mutex_lock (&cf->mutex);
461 if (!(block = cf_lookup (cf, no)))
463 zebra_mutex_unlock (&cf->mutex);
466 zebra_mutex_unlock (&cf->mutex);
467 if (!mf_read (cf->block_mf, block, offset, nbytes, buf))
469 yaz_log (YLOG_FATAL|YLOG_ERRNO, "cf_read no=" ZINT_FORMAT " block=" ZINT_FORMAT, no, block);
475 int cf_write (CFile cf, zint no, int offset, int nbytes, const void *buf)
480 zebra_mutex_lock (&cf->mutex);
481 if (!(block = cf_lookup (cf, no)))
483 block = cf_new (cf, no);
484 if (offset || nbytes)
486 mf_read (cf->rmf, no, 0, 0, cf->iobuf);
487 memcpy (cf->iobuf + offset, buf, nbytes);
493 zebra_mutex_unlock (&cf->mutex);
494 if (mf_write (cf->block_mf, block, offset, nbytes, buf))
496 yaz_log (YLOG_FATAL|YLOG_ERRNO, "cf_write no=" ZINT_FORMAT
497 " block=" ZINT_FORMAT, no, block);
503 int cf_close (CFile cf)
505 yaz_log (YLOG_DEBUG, "cf: close hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT
506 " total=" ZINT_FORMAT,
507 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
508 cf->head.next_bucket - cf->head.first_bucket);
509 flush_bucket (cf, -1);
512 mf_write (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
515 mf_close (cf->hash_mf);
516 mf_close (cf->block_mf);
520 zebra_mutex_destroy (&cf->mutex);