1 /* $Id: cfile.c,v 1.32 2004-12-08 12:23:08 adam Exp $
2 Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
31 static int write_head (CFile cf)
33 int left = cf->head.hash_size * sizeof(zint);
35 const char *tab = (char*) cf->array;
39 while (left >= (int) HASH_BSIZE)
41 mf_write (cf->hash_mf, bno++, 0, 0, tab);
46 mf_write (cf->hash_mf, bno, 0, left, tab);
50 static int read_head (CFile cf)
52 int left = cf->head.hash_size * sizeof(zint);
54 char *tab = (char*) cf->array;
58 while (left >= (int) HASH_BSIZE)
60 mf_read (cf->hash_mf, bno++, 0, 0, tab);
65 mf_read (cf->hash_mf, bno, 0, left, tab);
70 CFile cf_open (MFile mf, MFile_area area, const char *fname,
71 int block_size, int wflag, int *firstp)
75 CFile cf = (CFile) xmalloc (sizeof(*cf));
79 yaz_log (YLOG_DEBUG, "cf: open %s %s", cf->rmf->name, wflag ? "rdwr" : "rd");
80 sprintf (path, "%s-b", fname);
81 if (!(cf->block_mf = mf_open (area, path, block_size, wflag)))
83 yaz_log (YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
86 sprintf (path, "%s-i", fname);
87 if (!(cf->hash_mf = mf_open (area, path, HASH_BSIZE, wflag)))
89 yaz_log (YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
93 if (!mf_read (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head) ||
98 cf->head.block_size = block_size;
99 cf->head.hash_size = 199;
100 hash_bytes = cf->head.hash_size * sizeof(zint);
101 cf->head.flat_bucket = cf->head.next_bucket = cf->head.first_bucket =
102 (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2;
103 cf->head.next_block = 1;
105 mf_write (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
106 cf->array = (zint *) xmalloc (hash_bytes);
107 for (i = 0; i<cf->head.hash_size; i++)
115 assert (cf->head.block_size == block_size);
116 assert (cf->head.hash_size > 2);
117 hash_bytes = cf->head.hash_size * sizeof(zint);
118 assert (cf->head.next_bucket > 0);
119 assert (cf->head.next_block > 0);
120 if (cf->head.state == 1)
121 cf->array = (zint *) xmalloc (hash_bytes);
126 if (cf->head.state == 1)
128 cf->parray = (struct CFile_hash_bucket **)
129 xmalloc (cf->head.hash_size * sizeof(*cf->parray));
130 for (i = 0; i<cf->head.hash_size; i++)
131 cf->parray[i] = NULL;
135 cf->bucket_lru_front = cf->bucket_lru_back = NULL;
136 cf->bucket_in_memory = 0;
137 cf->max_bucket_in_memory = 100;
139 cf->iobuf = (char *) xmalloc (cf->head.block_size);
140 memset (cf->iobuf, 0, cf->head.block_size);
143 zebra_mutex_init (&cf->mutex);
147 static int cf_hash (CFile cf, zint no)
149 return (int) (((no >> 3) % cf->head.hash_size));
152 static void release_bucket (CFile cf, struct CFile_hash_bucket *p)
155 p->lru_prev->lru_next = p->lru_next;
157 cf->bucket_lru_back = p->lru_next;
159 p->lru_next->lru_prev = p->lru_prev;
161 cf->bucket_lru_front = p->lru_prev;
163 *p->h_prev = p->h_next;
165 p->h_next->h_prev = p->h_prev;
167 --(cf->bucket_in_memory);
171 static void flush_bucket (CFile cf, int no_to_flush)
174 struct CFile_hash_bucket *p;
176 for (i = 0; i != no_to_flush; i++)
178 p = cf->bucket_lru_back;
183 mf_write (cf->hash_mf, p->ph.this_bucket, 0, 0, &p->ph);
186 release_bucket (cf, p);
190 static struct CFile_hash_bucket *alloc_bucket (CFile cf, zint block_no, int hno)
192 struct CFile_hash_bucket *p, **pp;
194 if (cf->bucket_in_memory == cf->max_bucket_in_memory)
195 flush_bucket (cf, 1);
196 assert (cf->bucket_in_memory < cf->max_bucket_in_memory);
197 ++(cf->bucket_in_memory);
198 p = (struct CFile_hash_bucket *) xmalloc (sizeof(*p));
201 p->lru_prev = cf->bucket_lru_front;
202 if (cf->bucket_lru_front)
203 cf->bucket_lru_front->lru_next = p;
205 cf->bucket_lru_back = p;
206 cf->bucket_lru_front = p;
208 pp = cf->parray + hno;
212 (*pp)->h_prev = &p->h_next;
217 static struct CFile_hash_bucket *get_bucket (CFile cf, zint block_no, int hno)
219 struct CFile_hash_bucket *p;
221 p = alloc_bucket (cf, block_no, hno);
222 if (!mf_read (cf->hash_mf, block_no, 0, 0, &p->ph))
224 yaz_log (YLOG_FATAL|YLOG_ERRNO, "read get_bucket");
227 assert (p->ph.this_bucket == block_no);
232 static struct CFile_hash_bucket *new_bucket (CFile cf, zint *block_nop, int hno)
234 struct CFile_hash_bucket *p;
238 block_no = *block_nop = cf->head.next_bucket++;
239 p = alloc_bucket (cf, block_no, hno);
241 for (i = 0; i<HASH_BUCKET; i++)
246 p->ph.next_bucket = 0;
247 p->ph.this_bucket = block_no;
252 static zint cf_lookup_flat (CFile cf, zint no)
254 zint hno = (no*sizeof(zint))/HASH_BSIZE;
255 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
258 mf_read (cf->hash_mf, hno+cf->head.next_bucket, off, sizeof(zint), &vno);
262 static zint cf_lookup_hash (CFile cf, zint no)
264 int hno = cf_hash (cf, no);
265 struct CFile_hash_bucket *hb;
269 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
271 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
272 if (hb->ph.no[i] == no)
275 return hb->ph.vno[i];
278 for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket)
280 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
282 if (hb->ph.this_bucket == block_no)
288 /* extra check ... */
289 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
291 if (hb->ph.this_bucket == block_no)
293 yaz_log (YLOG_FATAL, "Found hash bucket on other chain (1)");
296 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
297 if (hb->ph.no[i] == no)
299 yaz_log (YLOG_FATAL, "Found hash bucket on other chain (2)");
305 hb = get_bucket (cf, block_no, hno);
306 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
307 if (hb->ph.no[i] == no)
308 return hb->ph.vno[i];
313 static void cf_write_flat (CFile cf, zint no, zint vno)
315 zint hno = (no*sizeof(zint))/HASH_BSIZE;
316 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
318 hno += cf->head.next_bucket;
319 if (hno >= cf->head.flat_bucket)
320 cf->head.flat_bucket = hno+1;
322 mf_write (cf->hash_mf, hno, off, sizeof(zint), &vno);
325 static void cf_moveto_flat (CFile cf)
327 struct CFile_hash_bucket *p;
331 yaz_log (YLOG_DEBUG, "cf: Moving to flat shadow: %s", cf->rmf->name);
332 yaz_log (YLOG_DEBUG, "cf: hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT " total="
334 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
335 cf->head.next_bucket - cf->head.first_bucket);
336 assert (cf->head.state == 1);
337 flush_bucket (cf, -1);
338 assert (cf->bucket_in_memory == 0);
339 p = (struct CFile_hash_bucket *) xmalloc (sizeof(*p));
340 for (i = cf->head.first_bucket; i < cf->head.next_bucket; i++)
342 if (!mf_read (cf->hash_mf, i, 0, 0, &p->ph))
344 yaz_log (YLOG_FATAL|YLOG_ERRNO, "read bucket moveto flat");
347 for (j = 0; j < HASH_BUCKET && p->ph.vno[j]; j++)
348 cf_write_flat (cf, p->ph.no[j], p->ph.vno[j]);
359 static zint cf_lookup (CFile cf, zint no)
361 if (cf->head.state > 1)
362 return cf_lookup_flat (cf, no);
363 return cf_lookup_hash (cf, no);
366 static zint cf_new_flat (CFile cf, zint no)
368 zint vno = (cf->head.next_block)++;
370 cf_write_flat (cf, no, vno);
374 static zint cf_new_hash (CFile cf, zint no)
376 int hno = cf_hash (cf, no);
377 struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno];
378 zint *bucketpp = &cf->array[hno];
380 zint vno = (cf->head.next_block)++;
382 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
383 if (!hb->ph.vno[HASH_BUCKET-1])
384 for (i = 0; i<HASH_BUCKET; i++)
396 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
397 if (hb->ph.this_bucket == *bucketpp)
399 bucketpp = &hb->ph.next_bucket;
407 /* extra check ... */
408 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
410 if (hb->ph.this_bucket == *bucketpp)
412 yaz_log (YLOG_FATAL, "Found hash bucket on other chain");
418 hb = get_bucket (cf, *bucketpp, hno);
420 for (i = 0; i<HASH_BUCKET; i++)
428 bucketpp = &hb->ph.next_bucket;
433 hb = new_bucket (cf, bucketpp, hno);
439 zint cf_new (CFile cf, zint no)
441 if (cf->head.state > 1)
442 return cf_new_flat (cf, no);
443 if (cf->no_miss*2 > cf->no_hits)
446 assert (cf->head.state > 1);
447 return cf_new_flat (cf, no);
449 return cf_new_hash (cf, no);
453 int cf_read (CFile cf, zint no, int offset, int nbytes, void *buf)
458 zebra_mutex_lock (&cf->mutex);
459 if (!(block = cf_lookup (cf, no)))
461 zebra_mutex_unlock (&cf->mutex);
464 zebra_mutex_unlock (&cf->mutex);
465 if (!mf_read (cf->block_mf, block, offset, nbytes, buf))
467 yaz_log (YLOG_FATAL|YLOG_ERRNO, "cf_read no=" ZINT_FORMAT " block=" ZINT_FORMAT, no, block);
473 int cf_write (CFile cf, zint no, int offset, int nbytes, const void *buf)
478 zebra_mutex_lock (&cf->mutex);
479 if (!(block = cf_lookup (cf, no)))
481 block = cf_new (cf, no);
482 if (offset || nbytes)
484 mf_read (cf->rmf, no, 0, 0, cf->iobuf);
485 memcpy (cf->iobuf + offset, buf, nbytes);
491 zebra_mutex_unlock (&cf->mutex);
492 if (mf_write (cf->block_mf, block, offset, nbytes, buf))
494 yaz_log (YLOG_FATAL|YLOG_ERRNO, "cf_write no=" ZINT_FORMAT
495 " block=" ZINT_FORMAT, no, block);
501 int cf_close (CFile cf)
503 yaz_log (YLOG_DEBUG, "cf: close hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT
504 " total=" ZINT_FORMAT,
505 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
506 cf->head.next_bucket - cf->head.first_bucket);
507 flush_bucket (cf, -1);
510 mf_write (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
513 mf_close (cf->hash_mf);
514 mf_close (cf->block_mf);
518 zebra_mutex_destroy (&cf->mutex);