2 * Copyright (C) 1994-2002, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
6 * $Id: extract.c,v 1.114 2002-04-04 14:14:13 adam Exp $
20 #if _FILE_OFFSET_BITS == 64
21 #define PRINTF_OFF_T "%Ld"
23 #define PRINTF_OFF_T "%ld"
26 static int records_inserted = 0;
27 static int records_updated = 0;
28 static int records_deleted = 0;
29 static int records_processed = 0;
31 static void logRecord (int showFlag)
35 if (showFlag || !(records_processed % 1000))
37 logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d",
38 records_processed, records_inserted, records_updated,
43 static void extract_init (struct recExtractCtrl *p, RecWord *w)
45 w->zebra_maps = p->zebra_maps;
47 w->attrSet = VAL_BIB1;
53 static const char **searchRecordKey (ZebraHandle zh,
54 struct recKeys *reckeys,
55 int attrSetS, int attrUseS)
57 static const char *ws[32];
69 for (i = 0; i<32; i++)
73 chS = zebraExplain_lookupSU (zh->reg->zei, attrSetS, attrUseS);
77 while (off < reckeys->buf_used)
80 const char *src = reckeys->buf + off;
88 memcpy (&ch, src, sizeof(ch));
94 memcpy (&attrSet, src, sizeof(attrSet));
95 src += sizeof(attrSet);
99 memcpy (&attrUse, src, sizeof(attrUse));
100 src += sizeof(attrUse);
107 seqno += ((lead>>2) & 15)-1;
110 memcpy (&seqno, src, sizeof(seqno));
111 src += sizeof(seqno);
117 attrUseS == attrUse && attrSetS == attrSet
126 woff = seqno - startSeq;
127 if (woff >= 0 && woff < 31)
131 off = src - reckeys->buf;
133 assert (off == reckeys->buf_used);
137 struct file_read_info {
138 off_t file_max; /* maximum offset so far */
139 off_t file_offset; /* current offset */
140 off_t file_moffset; /* offset of rec/rec boundary */
147 static struct file_read_info *file_read_start (int fd)
149 struct file_read_info *fi = (struct file_read_info *)
150 xmalloc (sizeof(*fi));
154 fi->file_moffset = 0;
160 static void file_read_stop (struct file_read_info *fi)
165 static off_t file_seek (void *handle, off_t offset)
167 struct file_read_info *p = (struct file_read_info *) handle;
168 p->file_offset = offset;
171 return lseek (p->fd, offset, SEEK_SET);
174 static off_t file_tell (void *handle)
176 struct file_read_info *p = (struct file_read_info *) handle;
177 return p->file_offset;
180 static int file_read (void *handle, char *buf, size_t count)
182 struct file_read_info *p = (struct file_read_info *) handle;
188 if (r > p->sdrmax - p->file_offset)
189 r = p->sdrmax - p->file_offset;
191 memcpy (buf, p->sdrbuf + p->file_offset, r);
194 r = read (fd, buf, count);
198 if (p->file_offset > p->file_max)
199 p->file_max = p->file_offset;
204 static void file_begin (void *handle)
206 struct file_read_info *p = (struct file_read_info *) handle;
208 p->file_offset = p->file_moffset;
209 if (!p->sdrbuf && p->file_moffset)
210 lseek (p->fd, p->file_moffset, SEEK_SET);
214 static void file_end (void *handle, off_t offset)
216 struct file_read_info *p = (struct file_read_info *) handle;
218 assert (p->file_more == 0);
220 p->file_moffset = offset;
223 static char *fileMatchStr (ZebraHandle zh,
224 struct recKeys *reckeys, struct recordGroup *rGroup,
225 const char *fname, const char *spec)
227 static char dstBuf[2048]; /* static here ??? */
229 const char *s = spec;
230 static const char **w;
234 while (*s == ' ' || *s == '\t')
240 char attset_str[64], attname_str[64];
241 data1_attset *attset;
244 int attSet = 1, attUse = 1;
248 for (i = 0; *s && *s != ',' && *s != ')'; s++)
250 attset_str[i++] = *s;
251 attset_str[i] = '\0';
256 for (i = 0; *s && *s != ')'; s++)
258 attname_str[i++] = *s;
259 attname_str[i] = '\0';
262 if ((attset = data1_get_attset (zh->reg->dh, attset_str)))
265 attSet = attset->reference;
266 att = data1_getattbyname(zh->reg->dh, attset, attname_str);
270 attUse = atoi (attname_str);
272 w = searchRecordKey (zh, reckeys, attSet, attUse);
277 for (i = 0; i<32; i++)
282 logf (LOG_WARN, "Missing ) in match criteria %s in group %s",
283 spec, rGroup->groupName ? rGroup->groupName : "none");
288 for (i = 0; i<32; i++)
289 if (matchFlag[i] && w[i])
301 logf (LOG_WARN, "Record didn't contain match"
302 " fields in (%s,%s)", attset_str, attname_str);
310 const char *spec_src = NULL;
311 const char *s1 = ++s;
312 while (*s1 && *s1 != ' ' && *s1 != '\t')
318 memcpy (special, s, spec_len);
319 special[spec_len] = '\0';
322 if (!strcmp (special, "group"))
323 spec_src = rGroup->groupName;
324 else if (!strcmp (special, "database"))
325 spec_src = rGroup->databaseName;
326 else if (!strcmp (special, "filename"))
328 else if (!strcmp (special, "type"))
329 spec_src = rGroup->recordType;
334 strcpy (dst, spec_src);
335 dst += strlen (spec_src);
338 else if (*s == '\"' || *s == '\'')
340 int stopMarker = *s++;
344 while (*s && *s != stopMarker)
347 tmpString[i++] = *s++;
352 strcpy (dst, tmpString);
353 dst += strlen (tmpString);
357 logf (LOG_WARN, "Syntax error in match criteria %s in group %s",
358 spec, rGroup->groupName ? rGroup->groupName : "none");
365 logf (LOG_WARN, "No match criteria for record %s in group %s",
366 fname, rGroup->groupName ? rGroup->groupName : "none");
373 struct recordLogInfo {
376 struct recordGroup *rGroup;
379 static void recordLogPreamble (int level, const char *msg, void *info)
381 struct recordLogInfo *p = (struct recordLogInfo *) info;
382 FILE *outf = yaz_log_file ();
386 fprintf (outf, "File %s, offset %d, type %s\n",
387 p->rGroup->recordType, p->recordOffset, p->fname);
388 log_event_start (NULL, NULL);
392 static int recordExtract (ZebraHandle zh,
393 SYSNO *sysno, const char *fname,
394 struct recordGroup *rGroup, int deleteFlag,
395 struct file_read_info *fi,
396 RecType recType, char *subType, void *clientData)
398 RecordAttr *recordAttr;
403 struct recordLogInfo logInfo;
404 off_t recordOffset = 0;
408 struct recExtractCtrl extractCtrl;
410 /* we are going to read from a file, so prepare the extraction */
413 zh->reg->keys.buf_used = 0;
414 zh->reg->keys.prevAttrUse = -1;
415 zh->reg->keys.prevAttrSet = -1;
416 zh->reg->keys.prevSeqNo = 0;
417 zh->reg->sortKeys = 0;
419 recordOffset = fi->file_moffset;
420 extractCtrl.offset = fi->file_moffset;
421 extractCtrl.readf = file_read;
422 extractCtrl.seekf = file_seek;
423 extractCtrl.tellf = file_tell;
424 extractCtrl.endf = file_end;
426 extractCtrl.subType = subType;
427 extractCtrl.init = extract_init;
428 extractCtrl.tokenAdd = extract_token_add;
429 extractCtrl.schemaAdd = extract_schema_add;
430 extractCtrl.dh = zh->reg->dh;
431 extractCtrl.handle = zh;
432 for (i = 0; i<256; i++)
434 if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
435 extractCtrl.seqno[i] = 1;
437 extractCtrl.seqno[i] = 0;
439 extractCtrl.zebra_maps = zh->reg->zebra_maps;
440 extractCtrl.flagShowRecords = !rGroup->flagRw;
443 printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset);
445 logInfo.fname = fname;
446 logInfo.recordOffset = recordOffset;
447 logInfo.rGroup = rGroup;
448 log_event_start (recordLogPreamble, &logInfo);
450 r = (*recType->extract)(clientData, &extractCtrl);
452 log_event_start (NULL, NULL);
454 if (r == RECCTRL_EXTRACT_EOF)
456 else if (r == RECCTRL_EXTRACT_ERROR)
458 /* error occured during extraction ... */
459 if (rGroup->flagRw &&
460 records_processed < rGroup->fileVerboseLimit)
462 logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType,
463 fname, recordOffset);
467 if (zh->reg->keys.buf_used == 0)
469 /* the extraction process returned no information - the record
470 is probably empty - unless flagShowRecords is in use */
474 logf (LOG_WARN, "empty %s %s " PRINTF_OFF_T, rGroup->recordType,
475 fname, recordOffset);
480 /* perform match if sysno not known and if match criteria is specified */
487 if (rGroup->recordId && *rGroup->recordId)
491 matchStr = fileMatchStr (zh, &zh->reg->keys, rGroup, fname,
495 rinfo = dict_lookup (zh->reg->matchDict, matchStr);
497 memcpy (sysno, rinfo+1, sizeof(*sysno));
501 logf (LOG_WARN, "Bad match criteria");
512 logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType,
513 fname, recordOffset);
514 logf (LOG_WARN, "cannot delete record above (seems new)");
517 if (records_processed < rGroup->fileVerboseLimit)
518 logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType,
519 fname, recordOffset);
520 rec = rec_new (zh->reg->records);
524 recordAttr = rec_init_attr (zh->reg->zei, rec);
528 dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno);
530 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
531 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
537 /* record already exists */
538 struct recKeys delkeys;
540 rec = rec_get (zh->reg->records, *sysno);
543 recordAttr = rec_init_attr (zh->reg->zei, rec);
545 if (recordAttr->runNumber ==
546 zebraExplain_runNumberIncrement (zh->reg->zei, 0))
548 logf (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, rGroup->recordType,
549 fname, recordOffset);
550 extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
555 delkeys.buf_used = rec->size[recInfo_delKeys];
556 delkeys.buf = rec->info[recInfo_delKeys];
557 extract_flushSortKeys (zh, *sysno, 0, &zh->reg->sortKeys);
558 extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
561 /* record going to be deleted */
562 if (!delkeys.buf_used)
564 logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T,
565 rGroup->recordType, fname, recordOffset);
566 logf (LOG_WARN, "cannot delete file above, storeKeys false");
570 if (records_processed < rGroup->fileVerboseLimit)
571 logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T,
572 rGroup->recordType, fname, recordOffset);
575 dict_delete (zh->reg->matchDict, matchStr);
576 rec_del (zh->reg->records, &rec);
584 /* record going to be updated */
585 if (!delkeys.buf_used)
587 logf (LOG_LOG, "update %s %s " PRINTF_OFF_T,
588 rGroup->recordType, fname, recordOffset);
589 logf (LOG_WARN, "cannot update file above, storeKeys false");
593 if (records_processed < rGroup->fileVerboseLimit)
594 logf (LOG_LOG, "update %s %s " PRINTF_OFF_T,
595 rGroup->recordType, fname, recordOffset);
596 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
601 /* update file type */
602 xfree (rec->info[recInfo_fileType]);
603 rec->info[recInfo_fileType] =
604 rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]);
606 /* update filename */
607 xfree (rec->info[recInfo_filename]);
608 rec->info[recInfo_filename] =
609 rec_strdup (fname, &rec->size[recInfo_filename]);
611 /* update delete keys */
612 xfree (rec->info[recInfo_delKeys]);
613 if (zh->reg->keys.buf_used > 0 && rGroup->flagStoreKeys == 1)
616 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
617 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
618 zh->reg->keys.buf = NULL;
619 zh->reg->keys.buf_max = 0;
621 rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used);
622 rec->size[recInfo_delKeys] = reckeys.buf_used;
623 memcpy (rec->info[recInfo_delKeys], reckeys.buf,
624 rec->size[recInfo_delKeys]);
629 rec->info[recInfo_delKeys] = NULL;
630 rec->size[recInfo_delKeys] = 0;
633 /* save file size of original record */
634 zebraExplain_recordBytesIncrement (zh->reg->zei,
635 - recordAttr->recordSize);
636 recordAttr->recordSize = fi->file_moffset - recordOffset;
637 if (!recordAttr->recordSize)
638 recordAttr->recordSize = fi->file_max - recordOffset;
639 zebraExplain_recordBytesIncrement (zh->reg->zei,
640 recordAttr->recordSize);
642 /* set run-number for this record */
643 recordAttr->runNumber = zebraExplain_runNumberIncrement (zh->reg->zei,
646 /* update store data */
647 xfree (rec->info[recInfo_storeData]);
648 if (rGroup->flagStoreData == 1)
650 rec->size[recInfo_storeData] = recordAttr->recordSize;
651 rec->info[recInfo_storeData] = (char *)
652 xmalloc (recordAttr->recordSize);
653 if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
655 logf (LOG_ERRNO|LOG_FATAL, "seek to " PRINTF_OFF_T " in %s",
656 recordOffset, fname);
659 if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize)
660 < recordAttr->recordSize)
662 logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
663 recordAttr->recordSize, fname);
669 rec->info[recInfo_storeData] = NULL;
670 rec->size[recInfo_storeData] = 0;
672 /* update database name */
673 xfree (rec->info[recInfo_databaseName]);
674 rec->info[recInfo_databaseName] =
675 rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]);
678 recordAttr->recordOffset = recordOffset;
680 /* commit this record */
681 rec_put (zh->reg->records, &rec);
686 int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
687 const struct recordGroup *rGroupP, int deleteFlag)
695 struct recordGroup rGroupM;
696 struct recordGroup *rGroup = &rGroupM;
697 struct file_read_info *fi;
700 memcpy (rGroup, rGroupP, sizeof(*rGroupP));
702 if (!rGroup->groupName || !*rGroup->groupName)
705 sprintf (gprefix, "%s.", rGroup->groupName);
707 logf (LOG_DEBUG, "fileExtract %s", fname);
709 /* determine file extension */
711 for (i = strlen(fname); --i >= 0; )
714 else if (fname[i] == '.')
716 strcpy (ext, fname+i+1);
719 /* determine file type - depending on extension */
720 if (!rGroup->recordType)
722 sprintf (ext_res, "%srecordType.%s", gprefix, ext);
723 if (!(rGroup->recordType = res_get (zh->res, ext_res)))
725 sprintf (ext_res, "%srecordType", gprefix);
726 rGroup->recordType = res_get (zh->res, ext_res);
729 if (!rGroup->recordType)
731 if (records_processed < rGroup->fileVerboseLimit)
732 logf (LOG_LOG, "? %s", fname);
735 if (!*rGroup->recordType)
738 recType_byName (zh->reg->recTypes, rGroup->recordType, subType,
741 logf (LOG_WARN, "No such record type: %s", rGroup->recordType);
745 /* determine match criteria */
746 if (!rGroup->recordId)
748 sprintf (ext_res, "%srecordId.%s", gprefix, ext);
749 rGroup->recordId = res_get (zh->res, ext_res);
752 /* determine database name */
753 if (!rGroup->databaseName)
755 sprintf (ext_res, "%sdatabase.%s", gprefix, ext);
756 if (!(rGroup->databaseName = res_get (zh->res, ext_res)))
758 sprintf (ext_res, "%sdatabase", gprefix);
759 rGroup->databaseName = res_get (zh->res, ext_res);
762 if (!rGroup->databaseName)
763 rGroup->databaseName = "Default";
765 /* determine if explain database */
767 sprintf (ext_res, "%sexplainDatabase", gprefix);
768 rGroup->explainDatabase =
769 atoi (res_get_def (zh->res, ext_res, "0"));
771 /* announce database */
772 if (zebraExplain_curDatabase (zh->reg->zei, rGroup->databaseName))
774 if (zebraExplain_newDatabase (zh->reg->zei, rGroup->databaseName,
775 rGroup->explainDatabase))
779 if (rGroup->flagStoreData == -1)
782 sprintf (ext_res, "%sstoreData.%s", gprefix, ext);
783 if (!(sval = res_get (zh->res, ext_res)))
785 sprintf (ext_res, "%sstoreData", gprefix);
786 sval = res_get (zh->res, ext_res);
789 rGroup->flagStoreData = atoi (sval);
791 if (rGroup->flagStoreData == -1)
792 rGroup->flagStoreData = 0;
794 if (rGroup->flagStoreKeys == -1)
798 sprintf (ext_res, "%sstoreKeys.%s", gprefix, ext);
799 sval = res_get (zh->res, ext_res);
802 sprintf (ext_res, "%sstoreKeys", gprefix);
803 sval = res_get (zh->res, ext_res);
806 sval = res_get (zh->res, "storeKeys");
808 rGroup->flagStoreKeys = atoi (sval);
810 if (rGroup->flagStoreKeys == -1)
811 rGroup->flagStoreKeys = 0;
813 if (sysno && deleteFlag)
817 if ((fd = open (fname, O_BINARY|O_RDONLY)) == -1)
819 logf (LOG_WARN|LOG_ERRNO, "open %s", fname);
823 fi = file_read_start (fd);
827 r = recordExtract (zh, sysno, fname, rGroup, deleteFlag, fi,
828 recType, subType, clientData);
829 } while (r && !sysno && fi->file_more);
837 int extract_rec_in_mem (ZebraHandle zh, const char *recordType,
838 const char *buf, size_t buf_size,
839 const char *databaseName, int delete_flag,
840 int test_mode, int *sysno,
841 int store_keys, int store_data,
842 const char *match_criteria)
844 RecordAttr *recordAttr;
845 struct recExtractCtrl extractCtrl;
851 const char *fname = "<no file>";
853 long recordOffset = 0;
854 struct zebra_fetch_control fc;
857 fc.record_int_buf = buf;
858 fc.record_int_len = buf_size;
859 fc.record_int_pos = 0;
861 fc.record_offset = 0;
863 extractCtrl.offset = 0;
864 extractCtrl.readf = zebra_record_int_read;
865 extractCtrl.seekf = zebra_record_int_seek;
866 extractCtrl.tellf = zebra_record_int_tell;
867 extractCtrl.endf = zebra_record_int_end;
868 extractCtrl.fh = &fc;
870 /* announce database */
871 if (zebraExplain_curDatabase (zh->reg->zei, databaseName))
873 if (zebraExplain_newDatabase (zh->reg->zei, databaseName, 0))
877 recType_byName (zh->reg->recTypes, recordType, subType,
880 logf (LOG_WARN, "No such record type: %s", recordType);
884 zh->reg->keys.buf_used = 0;
885 zh->reg->keys.prevAttrUse = -1;
886 zh->reg->keys.prevAttrSet = -1;
887 zh->reg->keys.prevSeqNo = 0;
888 zh->reg->sortKeys = 0;
890 extractCtrl.subType = subType;
891 extractCtrl.init = extract_init;
892 extractCtrl.tokenAdd = extract_token_add;
893 extractCtrl.schemaAdd = extract_schema_add;
894 extractCtrl.dh = zh->reg->dh;
895 extractCtrl.handle = zh;
896 extractCtrl.zebra_maps = zh->reg->zebra_maps;
897 extractCtrl.flagShowRecords = 0;
898 for (i = 0; i<256; i++)
900 if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
901 extractCtrl.seqno[i] = 1;
903 extractCtrl.seqno[i] = 0;
906 r = (*recType->extract)(clientData, &extractCtrl);
908 if (r == RECCTRL_EXTRACT_EOF)
910 else if (r == RECCTRL_EXTRACT_ERROR)
912 /* error occured during extraction ... */
914 yaz_log (LOG_WARN, "extract error");
916 if (rGroup->flagRw &&
917 records_processed < rGroup->fileVerboseLimit)
919 logf (LOG_WARN, "fail %s %s %ld", rGroup->recordType,
920 fname, (long) recordOffset);
925 if (zh->reg->keys.buf_used == 0)
927 /* the extraction process returned no information - the record
928 is probably empty - unless flagShowRecords is in use */
931 logf (LOG_WARN, "No keys generated for record");
932 logf (LOG_WARN, " The file is probably empty");
942 logf (LOG_LOG, "delete %s %s %ld", recordType,
943 fname, (long) recordOffset);
944 logf (LOG_WARN, "cannot delete record above (seems new)");
947 logf (LOG_LOG, "add %s %s %ld", recordType, fname,
948 (long) recordOffset);
949 rec = rec_new (zh->reg->records);
953 recordAttr = rec_init_attr (zh->reg->zei, rec);
957 dict_insert (zh->reg->matchDict, matchStr,
958 sizeof(*sysno), sysno);
960 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
961 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
965 /* record already exists */
966 struct recKeys delkeys;
968 rec = rec_get (zh->reg->records, *sysno);
971 recordAttr = rec_init_attr (zh->reg->zei, rec);
973 if (recordAttr->runNumber ==
974 zebraExplain_runNumberIncrement (zh->reg->zei, 0))
976 logf (LOG_LOG, "skipped %s %s %ld", recordType,
977 fname, (long) recordOffset);
981 delkeys.buf_used = rec->size[recInfo_delKeys];
982 delkeys.buf = rec->info[recInfo_delKeys];
983 extract_flushSortKeys (zh, *sysno, 0, &zh->reg->sortKeys);
984 extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
987 /* record going to be deleted */
988 if (!delkeys.buf_used)
990 logf (LOG_LOG, "delete %s %s %ld", recordType,
991 fname, (long) recordOffset);
992 logf (LOG_WARN, "cannot delete file above, storeKeys false");
996 logf (LOG_LOG, "delete %s %s %ld", recordType,
997 fname, (long) recordOffset);
1000 dict_delete (matchDict, matchStr);
1002 rec_del (zh->reg->records, &rec);
1009 /* record going to be updated */
1010 if (!delkeys.buf_used)
1012 logf (LOG_LOG, "update %s %s %ld", recordType,
1013 fname, (long) recordOffset);
1014 logf (LOG_WARN, "cannot update file above, storeKeys false");
1018 logf (LOG_LOG, "update %s %s %ld", recordType,
1019 fname, (long) recordOffset);
1020 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
1024 /* update file type */
1025 xfree (rec->info[recInfo_fileType]);
1026 rec->info[recInfo_fileType] =
1027 rec_strdup (recordType, &rec->size[recInfo_fileType]);
1029 /* update filename */
1030 xfree (rec->info[recInfo_filename]);
1031 rec->info[recInfo_filename] =
1032 rec_strdup (fname, &rec->size[recInfo_filename]);
1034 /* update delete keys */
1035 xfree (rec->info[recInfo_delKeys]);
1036 if (zh->reg->keys.buf_used > 0 && store_keys == 1)
1038 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
1039 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
1040 zh->reg->keys.buf = NULL;
1041 zh->reg->keys.buf_max = 0;
1045 rec->info[recInfo_delKeys] = NULL;
1046 rec->size[recInfo_delKeys] = 0;
1049 /* save file size of original record */
1050 zebraExplain_recordBytesIncrement (zh->reg->zei,
1051 - recordAttr->recordSize);
1053 recordAttr->recordSize = fi->file_moffset - recordOffset;
1054 if (!recordAttr->recordSize)
1055 recordAttr->recordSize = fi->file_max - recordOffset;
1057 recordAttr->recordSize = buf_size;
1059 zebraExplain_recordBytesIncrement (zh->reg->zei,
1060 recordAttr->recordSize);
1062 /* set run-number for this record */
1063 recordAttr->runNumber =
1064 zebraExplain_runNumberIncrement (zh->reg->zei, 0);
1066 /* update store data */
1067 xfree (rec->info[recInfo_storeData]);
1068 if (store_data == 1)
1070 rec->size[recInfo_storeData] = recordAttr->recordSize;
1071 rec->info[recInfo_storeData] = (char *)
1072 xmalloc (recordAttr->recordSize);
1074 memcpy (rec->info[recInfo_storeData], buf, recordAttr->recordSize);
1076 if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
1078 logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s",
1079 (long) recordOffset, fname);
1082 if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize)
1083 < recordAttr->recordSize)
1085 logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
1086 recordAttr->recordSize, fname);
1093 rec->info[recInfo_storeData] = NULL;
1094 rec->size[recInfo_storeData] = 0;
1096 /* update database name */
1097 xfree (rec->info[recInfo_databaseName]);
1098 rec->info[recInfo_databaseName] =
1099 rec_strdup (databaseName, &rec->size[recInfo_databaseName]);
1102 recordAttr->recordOffset = recordOffset;
1104 /* commit this record */
1105 rec_put (zh->reg->records, &rec);
1110 int explain_extract (void *handle, Record rec, data1_node *n)
1112 ZebraHandle zh = (ZebraHandle) handle;
1113 struct recExtractCtrl extractCtrl;
1116 if (zebraExplain_curDatabase (zh->reg->zei,
1117 rec->info[recInfo_databaseName]))
1120 if (zebraExplain_newDatabase (zh->reg->zei,
1121 rec->info[recInfo_databaseName], 0))
1125 zh->reg->keys.buf_used = 0;
1126 zh->reg->keys.prevAttrUse = -1;
1127 zh->reg->keys.prevAttrSet = -1;
1128 zh->reg->keys.prevSeqNo = 0;
1129 zh->reg->sortKeys = 0;
1131 extractCtrl.init = extract_init;
1132 extractCtrl.tokenAdd = extract_token_add;
1133 extractCtrl.schemaAdd = extract_schema_add;
1134 extractCtrl.dh = zh->reg->dh;
1135 for (i = 0; i<256; i++)
1136 extractCtrl.seqno[i] = 0;
1137 extractCtrl.zebra_maps = zh->reg->zebra_maps;
1138 extractCtrl.flagShowRecords = 0;
1139 extractCtrl.handle = handle;
1141 grs_extract_tree(&extractCtrl, n);
1143 if (rec->size[recInfo_delKeys])
1145 struct recKeys delkeys;
1146 struct sortKey *sortKeys = 0;
1148 delkeys.buf_used = rec->size[recInfo_delKeys];
1149 delkeys.buf = rec->info[recInfo_delKeys];
1150 extract_flushSortKeys (zh, rec->sysno, 0, &sortKeys);
1151 extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys);
1153 extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys);
1154 extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys);
1156 xfree (rec->info[recInfo_delKeys]);
1157 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
1158 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
1159 zh->reg->keys.buf = NULL;
1160 zh->reg->keys.buf_max = 0;
1164 void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
1165 int cmd, struct recKeys *reckeys)
1169 unsigned char attrSet = (unsigned char) -1;
1170 unsigned short attrUse = (unsigned short) -1;
1175 ZebraExplainInfo zei = zh->reg->zei;
1177 if (!zh->reg->key_buf)
1179 int mem = 8*1024*1024;
1180 zh->reg->key_buf = (char**) xmalloc (mem);
1181 zh->reg->ptr_top = mem/sizeof(char*);
1183 zh->reg->key_buf_used = 0;
1184 zh->reg->key_file_no = 0;
1186 zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1187 while (off < reckeys->buf_used)
1189 const char *src = reckeys->buf + off;
1198 memcpy (&ch, src, sizeof(ch));
1204 memcpy (&attrSet, src, sizeof(attrSet));
1205 src += sizeof(attrSet);
1209 memcpy (&attrUse, src, sizeof(attrUse));
1210 src += sizeof(attrUse);
1213 if (zh->reg->key_buf_used + 1024 >
1214 (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*))
1215 extract_flushWriteKeys (zh);
1217 (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] =
1218 (char*)zh->reg->key_buf + zh->reg->key_buf_used;
1221 ch = zebraExplain_lookupSU (zei, attrSet, attrUse);
1223 ch = zebraExplain_addSU (zei, attrSet, attrUse);
1226 zh->reg->key_buf_used +=
1227 key_SU_encode (ch,((char*)zh->reg->key_buf) +
1228 zh->reg->key_buf_used);
1231 ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++;
1233 ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = '\0';
1234 ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = cmd;
1237 seqno += ((lead>>2) & 15)-1;
1240 memcpy (&seqno, src, sizeof(seqno));
1241 src += sizeof(seqno);
1245 memcpy ((char*)zh->reg->key_buf + zh->reg->key_buf_used, &key, sizeof(key));
1246 (zh->reg->key_buf_used) += sizeof(key);
1247 off = src - reckeys->buf;
1249 assert (off == reckeys->buf_used);
1252 void extract_flushWriteKeys (ZebraHandle zh)
1255 char out_fname[200];
1257 struct encode_info encode_info;
1258 int ptr_i = zh->reg->ptr_i;
1262 if (!zh->reg->key_buf || ptr_i <= 0)
1265 (zh->reg->key_file_no)++;
1266 logf (LOG_LOG, "sorting section %d", (zh->reg->key_file_no));
1268 qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, sizeof(char*),
1270 extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no);
1272 if (!(outf = fopen (out_fname, "wb")))
1274 logf (LOG_FATAL|LOG_ERRNO, "fopen %s", out_fname);
1277 logf (LOG_LOG, "writing section %d", zh->reg->key_file_no);
1278 prevcp = cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i];
1280 encode_key_init (&encode_info);
1281 encode_key_write (cp, &encode_info, outf);
1285 cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i];
1286 if (strcmp (cp, prevcp))
1288 encode_key_init (&encode_info);
1289 encode_key_write (cp, &encode_info, outf);
1293 encode_key_write (cp + strlen(cp), &encode_info, outf);
1296 qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_x_compare);
1297 extract_get_fname_tmp (out_fname, key_file_no);
1299 if (!(outf = fopen (out_fname, "wb")))
1301 logf (LOG_FATAL|LOG_ERRNO, "fopen %s", out_fname);
1304 logf (LOG_LOG, "writing section %d", key_file_no);
1306 prevcp = key_buf[ptr_top-i];
1308 if (!--i || strcmp (prevcp, key_buf[ptr_top-i]))
1310 key_y_len = strlen(prevcp)+1;
1312 logf (LOG_LOG, "key_y_len: %2d %02x %02x %s",
1313 key_y_len, prevcp[0], prevcp[1], 2+prevcp);
1315 qsort (key_buf + ptr_top-ptr_i, ptr_i - i,
1316 sizeof(char*), key_y_compare);
1317 cp = key_buf[ptr_top-ptr_i];
1319 encode_key_init (&encode_info);
1320 encode_key_write (cp, &encode_info, outf);
1323 cp = key_buf[ptr_top-ptr_i];
1324 encode_key_write (cp+key_y_len, &encode_info, outf);
1328 prevcp = key_buf[ptr_top-ptr_i];
1333 logf (LOG_FATAL|LOG_ERRNO, "fclose %s", out_fname);
1336 logf (LOG_LOG, "finished section %d", zh->reg->key_file_no);
1338 zh->reg->key_buf_used = 0;
1341 void extract_add_index_string (RecWord *p, const char *string,
1345 unsigned char attrSet;
1346 unsigned short attrUse;
1349 int *pseqno = &p->seqnos[p->reg_type];
1350 ZebraHandle zh = p->extractCtrl->handle;
1351 ZebraExplainInfo zei = zh->reg->zei;
1352 struct recKeys *keys = &zh->reg->keys;
1354 if (keys->buf_used+1024 > keys->buf_max)
1358 b = (char *) xmalloc (keys->buf_max += 128000);
1359 if (keys->buf_used > 0)
1360 memcpy (b, keys->buf, keys->buf_used);
1364 dst = keys->buf + keys->buf_used;
1366 attrSet = p->attrSet;
1367 if (keys->buf_used > 0 && keys->prevAttrSet == attrSet)
1370 keys->prevAttrSet = attrSet;
1371 attrUse = p->attrUse;
1372 if (keys->buf_used > 0 && keys->prevAttrUse == attrUse)
1375 keys->prevAttrUse = attrUse;
1377 diff = 1 + *pseqno - keys->prevSeqNo;
1378 if (diff >= 1 && diff <= 15)
1379 lead |= (diff << 2);
1383 keys->prevSeqNo = *pseqno;
1390 int ch = zebraExplain_lookupSU (zei, attrSet, attrUse);
1393 ch = zebraExplain_addSU (zei, attrSet, attrUse);
1394 yaz_log (LOG_DEBUG, "addSU set=%d use=%d SU=%d",
1395 attrSet, attrUse, ch);
1398 memcpy (dst, &ch, sizeof(ch));
1404 memcpy (dst, &attrSet, sizeof(attrSet));
1405 dst += sizeof(attrSet);
1409 memcpy (dst, &attrUse, sizeof(attrUse));
1410 dst += sizeof(attrUse);
1413 *dst++ = p->reg_type;
1414 memcpy (dst, string, length);
1420 memcpy (dst, pseqno, sizeof(*pseqno));
1421 dst += sizeof(*pseqno);
1423 keys->buf_used = dst - keys->buf;
1428 static void extract_add_sort_string (RecWord *p, const char *string,
1432 ZebraHandle zh = p->extractCtrl->handle;
1434 for (sk = zh->reg->sortKeys; sk; sk = sk->next)
1435 if (sk->attrSet == p->attrSet && sk->attrUse == p->attrUse)
1438 sk = (struct sortKey *) xmalloc (sizeof(*sk));
1439 sk->next = zh->reg->sortKeys;
1440 zh->reg->sortKeys = sk;
1442 sk->string = (char *) xmalloc (length);
1443 sk->length = length;
1444 memcpy (sk->string, string, length);
1446 sk->attrSet = p->attrSet;
1447 sk->attrUse = p->attrUse;
1450 void extract_add_string (RecWord *p, const char *string, int length)
1452 assert (length > 0);
1453 if (zebra_maps_is_sort (p->zebra_maps, p->reg_type))
1454 extract_add_sort_string (p, string, length);
1456 extract_add_index_string (p, string, length);
1459 static void extract_add_incomplete_field (RecWord *p)
1461 const char *b = p->string;
1462 int remain = p->length;
1463 const char **map = 0;
1466 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain);
1470 char buf[IT_MAX_WORD+1];
1474 while (map && *map && **map == *CHR_SPACE)
1476 remain = p->length - (b - p->string);
1478 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain);
1485 while (map && *map && **map != *CHR_SPACE)
1487 const char *cp = *map;
1489 while (i < IT_MAX_WORD && *cp)
1491 remain = p->length - (b - p->string);
1493 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain);
1499 extract_add_string (p, buf, i);
1501 (p->seqnos[p->reg_type])++; /* to separate this from next one */
1504 static void extract_add_complete_field (RecWord *p)
1506 const char *b = p->string;
1507 char buf[IT_MAX_WORD+1];
1508 const char **map = 0;
1509 int i = 0, remain = p->length;
1512 map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain);
1514 while (remain > 0 && i < IT_MAX_WORD)
1516 while (map && *map && **map == *CHR_SPACE)
1518 remain = p->length - (b - p->string);
1520 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain);
1527 if (i && i < IT_MAX_WORD)
1528 buf[i++] = *CHR_SPACE;
1529 while (map && *map && **map != *CHR_SPACE)
1531 const char *cp = *map;
1533 if (i >= IT_MAX_WORD)
1535 while (i < IT_MAX_WORD && *cp)
1537 remain = p->length - (b - p->string);
1539 map = zebra_maps_input (p->zebra_maps, p->reg_type, &b,
1547 extract_add_string (p, buf, i);
1550 void extract_token_add (RecWord *p)
1553 if ((wrbuf = zebra_replace(p->zebra_maps, p->reg_type, 0,
1554 p->string, p->length)))
1556 p->string = wrbuf_buf(wrbuf);
1557 p->length = wrbuf_len(wrbuf);
1559 if (zebra_maps_is_complete (p->zebra_maps, p->reg_type))
1560 extract_add_complete_field (p);
1562 extract_add_incomplete_field(p);
1565 void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid)
1567 ZebraHandle zh = (ZebraHandle) (p->handle);
1568 zebraExplain_addSchema (zh->reg->zei, oid);
1571 void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
1572 int cmd, struct sortKey **skp)
1574 struct sortKey *sk = *skp;
1575 SortIdx sortIdx = zh->reg->sortIdx;
1577 sortIdx_sysno (sortIdx, sysno);
1580 struct sortKey *sk_next = sk->next;
1581 sortIdx_type (sortIdx, sk->attrUse);
1582 sortIdx_add (sortIdx, sk->string, sk->length);
1590 void encode_key_init (struct encode_info *i)
1597 char *encode_key_int (int d, char *bp)
1601 else if (d <= 16383)
1603 *bp++ = 64 + (d>>8);
1606 else if (d <= 4194303)
1608 *bp++ = 128 + (d>>16);
1609 *bp++ = (d>>8) & 255;
1614 *bp++ = 192 + (d>>24);
1615 *bp++ = (d>>16) & 255;
1616 *bp++ = (d>>8) & 255;
1622 void encode_key_write (char *k, struct encode_info *i, FILE *outf)
1627 while ((*bp++ = *k++))
1629 memcpy (&key, k+1, sizeof(struct it_key));
1630 bp = encode_key_int ( (key.sysno - i->sysno) * 2 + *k, bp);
1631 if (i->sysno != key.sysno)
1633 i->sysno = key.sysno;
1636 else if (!i->seqno && !key.seqno && i->cmd == *k)
1638 bp = encode_key_int (key.seqno - i->seqno, bp);
1639 i->seqno = key.seqno;
1641 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1643 logf (LOG_FATAL|LOG_ERRNO, "fwrite");