2 * Copyright (C) 1994-2002, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
6 * $Id: mfile.c,v 1.44 2002-04-11 20:09:08 adam Exp $
11 * TODO: The size estimates in init may not be accurate due to
12 * only partially written final blocks.
15 #include <sys/types.h>
29 #include <zebra-lock.h>
33 static int scan_areadef(MFile_area ma, const char *ad, const char *base)
36 * If no definition is given, use current directory, unlimited.
38 char dirname[FILENAME_MAX+1];
39 mf_dir **dp = &ma->dirs, *dir = *dp;
46 int i = 0, fact = 1, multi;
49 while (*ad == ' ' || *ad == '\t')
53 if (!yaz_is_abspath(ad) && base)
55 strcpy (dirname, base);
61 if (*ad == ':' && strchr ("+-0123456789", ad[1]))
70 logf (LOG_WARN, "Missing colon after path: %s", ad0);
75 logf (LOG_WARN, "Empty path: %s", ad0);
78 while (*ad == ' ' || *ad == '\t')
88 if (*ad < '0' || *ad > '9')
90 logf (LOG_FATAL, "Missing size after path: %s", ad0);
94 while (*ad >= '0' && *ad <= '9')
95 size = size*10 + (*ad++ - '0');
98 case 'B': case 'b': multi = 1; break;
99 case 'K': case 'k': multi = 1024; break;
100 case 'M': case 'm': multi = 1048576; break;
101 case 'G': case 'g': multi = 1073741824; break;
103 logf (LOG_FATAL, "Missing unit: %s", ad0);
106 logf (LOG_FATAL, "Illegal unit: %c in %s", *ad, ad0);
110 *dp = dir = (mf_dir *) xmalloc(sizeof(mf_dir));
112 strcpy(dir->name, dirname);
113 dir->max_bytes = dir->avail_bytes = fact * size * multi;
119 static int file_position(MFile mf, int pos, int offset)
121 int off = 0, c = mf->cur_file, ps;
123 if ((c > 0 && pos <= mf->files[c-1].top) ||
124 (c < mf->no_files -1 && pos > mf->files[c].top))
127 while (c + 1 < mf->no_files && mf->files[c].top < pos)
129 off += mf->files[c].blocks;
132 assert(c < mf->no_files);
135 off = c ? (mf->files[c-1].top + 1) : 0;
136 if (mf->files[c].fd < 0 && (mf->files[c].fd = open(mf->files[c].path,
137 mf->wr ? (O_BINARY|O_RDWR|O_CREAT) : (O_BINARY|O_RDONLY), 0666)) < 0)
139 if (!mf->wr && errno == ENOENT && off == 0)
141 logf (LOG_WARN|LOG_ERRNO, "Failed to open %s", mf->files[c].path);
144 if (lseek(mf->files[c].fd, (ps = pos - off) * mf->blocksize + offset,
147 logf (LOG_WARN|LOG_ERRNO, "Failed to seek in %s", mf->files[c].path);
154 static int cmp_part_file(const void *p1, const void *p2)
156 return ((part_file *)p1)->number - ((part_file *)p2)->number;
160 * Create a new area, cotaining metafiles in directories.
161 * Find the part-files in each directory, and inventory the existing metafiles.
163 MFile_area mf_init(const char *name, const char *spec, const char *base)
165 MFile_area ma = (MFile_area) xmalloc(sizeof(*ma));
168 part_file *part_f = 0;
172 char metaname[FILENAME_MAX+1], tmpnam[FILENAME_MAX+1];
174 logf (LOG_DEBUG, "mf_init(%s)", name);
175 strcpy(ma->name, name);
178 if (scan_areadef(ma, spec, base) < 0)
180 logf (LOG_WARN, "Failed to access description of '%s'", name);
183 /* look at each directory */
184 for (dirp = ma->dirs; dirp; dirp = dirp->next)
186 if (!(dd = opendir(dirp->name)))
188 logf (LOG_WARN|LOG_ERRNO, "Failed to open directory %s",
192 /* look at each file */
193 while ((dent = readdir(dd)))
195 int len = strlen(dent->d_name);
196 const char *cp = strrchr (dent->d_name, '-');
197 if (strchr (".-", *dent->d_name))
199 if (len < 5 || !cp || strcmp (dent->d_name + len - 3, ".mf"))
202 memcpy (metaname, dent->d_name, cp - dent->d_name);
203 metaname[ cp - dent->d_name] = '\0';
205 for (meta_f = ma->mfiles; meta_f; meta_f = meta_f->next)
208 if (!strcmp(meta_f->name, metaname))
210 part_f = &meta_f->files[meta_f->no_files++];
217 meta_f = (meta_file *) xmalloc(sizeof(*meta_f));
218 zebra_mutex_init (&meta_f->mutex);
220 meta_f->next = ma->mfiles;
222 meta_f->cur_file = -1;
224 strcpy(meta_f->name, metaname);
225 part_f = &meta_f->files[0];
226 meta_f->no_files = 1;
228 part_f->number = number;
231 sprintf(tmpnam, "%s/%s", dirp->name, dent->d_name);
232 part_f->path = xstrdup(tmpnam);
234 if ((fd = open(part_f->path, O_BINARY|O_RDONLY)) < 0)
236 logf (LOG_FATAL|LOG_ERRNO, "Failed to access %s",
240 if ((part_f->bytes = lseek(fd, 0, SEEK_END)) < 0)
242 logf (LOG_FATAL|LOG_ERRNO, "Failed to seek in %s",
247 if (dirp->max_bytes >= 0)
248 dirp->avail_bytes -= part_f->bytes;
252 for (meta_f = ma->mfiles; meta_f; meta_f = meta_f->next)
254 logf (LOG_DEBUG, "mf_init: %s consists of %d part(s)", meta_f->name,
256 qsort(meta_f->files, meta_f->no_files, sizeof(part_file),
262 void mf_destroy(MFile_area ma)
280 meta_file *m = meta_f;
282 for (i = 0; i<m->no_files; i++)
284 xfree (m->files[i].path);
286 zebra_mutex_destroy (&meta_f->mutex);
287 meta_f = meta_f->next;
293 void mf_reset(MFile_area ma)
303 meta_file *m = meta_f;
306 for (i = 0; i<m->no_files; i++)
308 unlink (m->files[i].path);
309 xfree (m->files[i].path);
311 meta_f = meta_f->next;
319 * If !ma, Use MF_DEFAULT_AREA.
321 MFile mf_open(MFile_area ma, const char *name, int block_size, int wflag)
325 char tmp[FILENAME_MAX+1];
328 logf(LOG_DEBUG, "mf_open(%s bs=%d, %s)", name, block_size,
329 wflag ? "RW" : "RDONLY");
331 for (mnew = ma->mfiles; mnew; mnew = mnew->next)
332 if (!strcmp(name, mnew->name))
341 mnew = (meta_file *) xmalloc(sizeof(*mnew));
342 strcpy(mnew->name, name);
343 /* allocate one, empty file */
344 zebra_mutex_init (&mnew->mutex);
346 mnew->files[0].bytes = 0;
347 mnew->files[0].blocks = 0;
348 mnew->files[0].top = -1;
349 mnew->files[0].number = 0;
350 mnew->files[0].fd = -1;
351 mnew->min_bytes_creat = MF_MIN_BLOCKS_CREAT * block_size;
352 for (dp = ma->dirs; dp && dp->max_bytes >= 0 && dp->avail_bytes <
353 mnew->min_bytes_creat; dp = dp->next);
356 logf (LOG_FATAL, "Insufficient space for new mfile.");
359 mnew->files[0].dir = dp;
360 sprintf(tmp, "%s/%s-%d.mf", dp->name, mnew->name, 0);
361 mnew->files[0].path = xstrdup(tmp);
363 mnew->next = ma->mfiles;
368 for (i = 0; i < mnew->no_files; i++)
370 if (mnew->files[i].bytes % block_size)
371 mnew->files[i].bytes += block_size - mnew->files[i].bytes %
373 mnew->files[i].blocks = mnew->files[i].bytes / block_size;
377 mnew->blocksize = block_size;
378 mnew->min_bytes_creat = MF_MIN_BLOCKS_CREAT * block_size;
383 for (i = 0; i < mnew->no_files; i++)
385 mnew->files[i].blocks = mnew->files[i].bytes / mnew->blocksize;
386 if (i == mnew->no_files - 1)
387 mnew->files[i].top = -1;
390 i ? (mnew->files[i-1].top + mnew->files[i].blocks)
391 : (mnew->files[i].blocks - 1);
399 int mf_close(MFile mf)
403 logf (LOG_DEBUG, "mf_close(%s)", mf->name);
405 for (i = 0; i < mf->no_files; i++)
406 if (mf->files[i].fd >= 0)
408 close(mf->files[i].fd);
409 mf->files[i].fd = -1;
416 * Read one block from a metafile. Interface mirrors bfile.
418 int mf_read(MFile mf, int no, int offset, int nbytes, void *buf)
422 zebra_mutex_lock (&mf->mutex);
423 if ((rd = file_position(mf, no, offset)) < 0)
427 zebra_mutex_unlock (&mf->mutex);
433 toread = nbytes ? nbytes : mf->blocksize;
434 if ((rd = read(mf->files[mf->cur_file].fd, buf, toread)) < 0)
436 logf (LOG_FATAL|LOG_ERRNO, "mf_read: Read failed (%s)",
437 mf->files[mf->cur_file].path);
440 zebra_mutex_unlock (&mf->mutex);
450 int mf_write(MFile mf, int no, int offset, int nbytes, const void *buf)
452 int ps, nblocks, towrite;
454 char tmp[FILENAME_MAX+1];
455 unsigned char dummych = '\xff';
457 zebra_mutex_lock (&mf->mutex);
458 if ((ps = file_position(mf, no, offset)) < 0)
460 /* file needs to grow */
461 while (ps >= mf->files[mf->cur_file].blocks)
463 /* file overflow - allocate new file */
464 if (mf->files[mf->cur_file].dir->max_bytes >= 0 &&
465 (ps - mf->files[mf->cur_file].blocks + 1) * mf->blocksize >
466 mf->files[mf->cur_file].dir->avail_bytes)
469 if ((nblocks = mf->files[mf->cur_file].dir->avail_bytes /
472 logf (LOG_DEBUG, "Capping off file %s at pos %d",
473 mf->files[mf->cur_file].path, nblocks);
474 if ((ps = file_position(mf,
475 (mf->cur_file ? mf->files[mf->cur_file-1].top : 0) +
476 mf->files[mf->cur_file].blocks + nblocks - 1, 0)) < 0)
478 logf (LOG_DEBUG, "ps = %d", ps);
479 if (write(mf->files[mf->cur_file].fd, &dummych, 1) < 1)
481 logf (LOG_ERRNO|LOG_FATAL, "write dummy");
484 mf->files[mf->cur_file].blocks += nblocks;
485 mf->files[mf->cur_file].bytes += nblocks * mf->blocksize;
486 mf->files[mf->cur_file].dir->avail_bytes -= nblocks *
490 logf (LOG_DEBUG, "Creating new file.");
491 for (dp = mf->ma->dirs; dp && dp->max_bytes >= 0 &&
492 dp->avail_bytes < mf->min_bytes_creat; dp = dp->next);
495 logf (LOG_FATAL, "Cannot allocate more space for %s",
499 mf->files[mf->cur_file].top = (mf->cur_file ?
500 mf->files[mf->cur_file-1].top : -1) +
501 mf->files[mf->cur_file].blocks;
502 mf->files[++(mf->cur_file)].top = -1;
503 mf->files[mf->cur_file].dir = dp;
504 mf->files[mf->cur_file].number =
505 mf->files[mf->cur_file-1].number + 1;
506 mf->files[mf->cur_file].blocks =
507 mf->files[mf->cur_file].bytes = 0;
508 mf->files[mf->cur_file].fd = -1;
509 sprintf(tmp, "%s/%s-%d.mf", dp->name, mf->name,
510 mf->files[mf->cur_file].number);
511 mf->files[mf->cur_file].path = xstrdup(tmp);
513 /* open new file and position at beginning */
514 if ((ps = file_position(mf, no, offset)) < 0)
519 nblocks = ps - mf->files[mf->cur_file].blocks + 1;
520 mf->files[mf->cur_file].blocks += nblocks;
521 mf->files[mf->cur_file].bytes += nblocks * mf->blocksize;
522 if (mf->files[mf->cur_file].dir->max_bytes >= 0)
523 mf->files[mf->cur_file].dir->avail_bytes -=
524 nblocks * mf->blocksize;
527 towrite = nbytes ? nbytes : mf->blocksize;
528 if (write(mf->files[mf->cur_file].fd, buf, towrite) < towrite)
530 logf (LOG_FATAL|LOG_ERRNO, "Write failed for file %s part %d",
531 mf->name, mf->cur_file);
534 zebra_mutex_unlock (&mf->mutex);
539 * Destroy a metafile, unlinking component files. File must be open.
541 int mf_unlink(MFile mf)
545 for (i = 0; i < mf->no_files; i++)
546 unlink (mf->files[i].path);
551 * Unlink the file by name, rather than MFile-handle. File should be closed.
553 int mf_unlink_name(MFile_area ma, const char *name)