1 /* $Id: extract.c,v 1.182 2005-04-29 10:33:53 adam Exp $
2 Copyright (C) 1995-2005
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
37 #if _FILE_OFFSET_BITS == 64
38 #define PRINTF_OFF_T "%Ld"
40 #define PRINTF_OFF_T "%ld"
43 #define USE_SHELLSORT 0
46 static void shellsort(void *ar, int r, size_t s,
47 int (*cmp)(const void *a, const void *b))
52 static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936,
53 13776, 4592, 1968, 861, 336,
54 112, 48, 21, 7, 3, 1 };
55 for ( k = 0; k < 16; k++)
56 for (h = incs[k], i = h; i < r; i++)
60 while (j > h && (*cmp)(a + s*(j-h), v) > 0)
62 memcpy (a + s*j, a + s*(j-h), s);
70 static void logRecord (ZebraHandle zh)
72 ++zh->records_processed;
73 if (!(zh->records_processed % 1000))
75 yaz_log (YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
76 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
77 zh->records_processed, zh->records_inserted, zh->records_updated,
82 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
84 static void extract_init (struct recExtractCtrl *p, RecWord *w)
86 w->zebra_maps = p->zebra_maps;
88 w->attrSet = VAL_BIB1;
97 static const char **searchRecordKey (ZebraHandle zh,
98 struct recKeys *reckeys,
99 int attrSetS, int attrUseS)
101 static const char *ws[32];
102 void *decode_handle = iscz1_start();
108 for (i = 0; i<32; i++)
111 while (off < reckeys->buf_used)
113 const char *src = reckeys->buf + off;
115 char *dst = (char*) &key;
116 int attrSet, attrUse;
118 iscz1_decode(decode_handle, &dst, &src);
119 assert(key.len < 4 && key.len > 2);
121 attrSet = (int) key.mem[0] >> 16;
122 attrUse = (int) key.mem[0] & 65535;
123 seqno = (int) key.mem[key.len-1];
125 if (attrUseS == attrUse && attrSetS == attrSet)
131 woff = seqno - startSeq;
132 if (woff >= 0 && woff < 31)
138 off = src - reckeys->buf;
140 iscz1_stop(decode_handle);
141 assert (off == reckeys->buf_used);
145 struct file_read_info {
146 off_t file_max; /* maximum offset so far */
147 off_t file_offset; /* current offset */
148 off_t file_moffset; /* offset of rec/rec boundary */
153 static struct file_read_info *file_read_start (int fd)
155 struct file_read_info *fi = (struct file_read_info *)
156 xmalloc (sizeof(*fi));
160 fi->file_moffset = 0;
166 static void file_read_stop (struct file_read_info *fi)
171 static off_t file_seek (void *handle, off_t offset)
173 struct file_read_info *p = (struct file_read_info *) handle;
174 p->file_offset = offset;
175 return lseek (p->fd, offset, SEEK_SET);
178 static off_t file_tell (void *handle)
180 struct file_read_info *p = (struct file_read_info *) handle;
181 return p->file_offset;
184 static int file_read (void *handle, char *buf, size_t count)
186 struct file_read_info *p = (struct file_read_info *) handle;
189 r = read (fd, buf, count);
193 if (p->file_offset > p->file_max)
194 p->file_max = p->file_offset;
199 static void file_end (void *handle, off_t offset)
201 struct file_read_info *p = (struct file_read_info *) handle;
203 if (offset != p->file_moffset)
205 p->file_moffset = offset;
210 static char *fileMatchStr (ZebraHandle zh,
211 struct recKeys *reckeys,
212 const char *fname, const char *spec)
214 static char dstBuf[2048]; /* static here ??? */
216 const char *s = spec;
217 static const char **w;
221 while (*s == ' ' || *s == '\t')
227 char attset_str[64], attname_str[64];
228 data1_attset *attset;
231 int attSet = 1, attUse = 1;
235 for (i = 0; *s && *s != ',' && *s != ')'; s++)
237 attset_str[i++] = *s;
238 attset_str[i] = '\0';
243 for (i = 0; *s && *s != ')'; s++)
245 attname_str[i++] = *s;
246 attname_str[i] = '\0';
249 if ((attset = data1_get_attset (zh->reg->dh, attset_str)))
252 attSet = attset->reference;
253 att = data1_getattbyname(zh->reg->dh, attset, attname_str);
257 attUse = atoi (attname_str);
259 w = searchRecordKey (zh, reckeys, attSet, attUse);
264 for (i = 0; i<32; i++)
269 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
270 spec, zh->m_group ? zh->m_group : "none");
275 for (i = 0; i<32; i++)
276 if (matchFlag[i] && w[i])
288 yaz_log (YLOG_WARN, "Record didn't contain match"
289 " fields in (%s,%s)", attset_str, attname_str);
297 const char *spec_src = NULL;
298 const char *s1 = ++s;
299 while (*s1 && *s1 != ' ' && *s1 != '\t')
305 memcpy (special, s, spec_len);
306 special[spec_len] = '\0';
309 if (!strcmp (special, "group"))
310 spec_src = zh->m_group;
311 else if (!strcmp (special, "database"))
312 spec_src = zh->basenames[0];
313 else if (!strcmp (special, "filename")) {
316 else if (!strcmp (special, "type"))
317 spec_src = zh->m_record_type;
322 strcpy (dst, spec_src);
323 dst += strlen (spec_src);
326 else if (*s == '\"' || *s == '\'')
328 int stopMarker = *s++;
332 while (*s && *s != stopMarker)
335 tmpString[i++] = *s++;
340 strcpy (dst, tmpString);
341 dst += strlen (tmpString);
345 yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
346 spec, zh->m_group ? zh->m_group : "none");
353 yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
354 fname, zh->m_group ? zh->m_group : "none");
361 struct recordLogInfo {
364 struct recordGroup *rGroup;
367 void create_rec_keys_codec(struct recKeys *keys)
370 iscz1_reset(keys->codec_handle);
373 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
376 for (i = 0; i<256; i++)
378 if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
383 ctrl->zebra_maps = zh->reg->zebra_maps;
384 ctrl->flagShowRecords = !zh->m_flag_rw;
387 static int file_extract_record(ZebraHandle zh,
388 SYSNO *sysno, const char *fname,
390 struct file_read_info *fi,
393 void *recTypeClientData)
395 RecordAttr *recordAttr;
397 const char *matchStr = 0;
400 off_t recordOffset = 0;
402 /* announce database */
403 if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
405 if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
406 zh->m_explain_database))
412 struct recExtractCtrl extractCtrl;
414 /* we are going to read from a file, so prepare the extraction */
415 create_rec_keys_codec(&zh->reg->keys);
417 zh->reg->sortKeys.buf_used = 0;
419 recordOffset = fi->file_moffset;
420 extractCtrl.handle = zh;
421 extractCtrl.offset = fi->file_moffset;
422 extractCtrl.readf = file_read;
423 extractCtrl.seekf = file_seek;
424 extractCtrl.tellf = file_tell;
425 extractCtrl.endf = file_end;
427 extractCtrl.init = extract_init;
428 extractCtrl.tokenAdd = extract_token_add;
429 extractCtrl.schemaAdd = extract_schema_add;
430 extractCtrl.dh = zh->reg->dh;
431 extractCtrl.match_criteria[0] = '\0';
432 extractCtrl.first_record = fi->file_offset ? 0 : 1;
434 extract_set_store_data_prepare(&extractCtrl);
436 init_extractCtrl(zh, &extractCtrl);
439 printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset);
443 sprintf (msg, "%s:" PRINTF_OFF_T , fname, recordOffset);
444 yaz_log_init_prefix2 (msg);
447 r = (*recType->extract)(recTypeClientData, &extractCtrl);
449 yaz_log_init_prefix2 (0);
450 if (r == RECCTRL_EXTRACT_EOF)
452 else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
454 /* error occured during extraction ... */
456 zh->records_processed < zh->m_file_verbose_limit)
458 yaz_log (YLOG_WARN, "fail %s %s " PRINTF_OFF_T, zh->m_record_type,
459 fname, recordOffset);
463 else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
465 /* error occured during extraction ... */
467 zh->records_processed < zh->m_file_verbose_limit)
469 yaz_log (YLOG_WARN, "no filter for %s %s "
470 PRINTF_OFF_T, zh->m_record_type,
471 fname, recordOffset);
475 if (extractCtrl.match_criteria[0])
476 matchStr = extractCtrl.match_criteria;
479 /* perform match if sysno not known and if match criteria is specified */
485 if (matchStr == 0 && zh->m_record_id && *zh->m_record_id)
488 matchStr = fileMatchStr (zh, &zh->reg->keys, fname,
492 yaz_log(YLOG_WARN, "Bad match criteria");
498 char *rinfo = dict_lookup (zh->reg->matchDict, matchStr);
501 assert(*rinfo == sizeof(*sysno));
502 memcpy (sysno, rinfo+1, sizeof(*sysno));
506 if (! *sysno && zh->reg->keys.buf_used == 0)
508 /* the extraction process returned no information - the record
509 is probably empty - unless flagShowRecords is in use */
513 if (zh->records_processed < zh->m_file_verbose_limit)
514 yaz_log (YLOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type,
515 fname, recordOffset);
524 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, zh->m_record_type,
525 fname, recordOffset);
526 yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
529 if (zh->records_processed < zh->m_file_verbose_limit)
530 yaz_log (YLOG_LOG, "add %s %s " PRINTF_OFF_T, zh->m_record_type,
531 fname, recordOffset);
532 rec = rec_new (zh->reg->records);
536 recordAttr = rec_init_attr (zh->reg->zei, rec);
540 dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno);
542 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
543 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
545 zh->records_inserted++;
549 /* record already exists */
550 struct recKeys delkeys;
551 struct sortKeys sortKeys;
553 rec = rec_get (zh->reg->records, *sysno);
556 recordAttr = rec_init_attr (zh->reg->zei, rec);
558 if (!force_update && recordAttr->runNumber ==
559 zebraExplain_runNumberIncrement (zh->reg->zei, 0))
561 yaz_log (YLOG_LOG, "run number = " ZINT_FORMAT,
562 recordAttr->runNumber);
563 yaz_log (YLOG_LOG, "skipped %s %s " PRINTF_OFF_T,
564 zh->m_record_type, fname, recordOffset);
565 extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
570 delkeys.buf_used = rec->size[recInfo_delKeys];
571 delkeys.buf = rec->info[recInfo_delKeys];
573 sortKeys.buf_used = rec->size[recInfo_sortKeys];
574 sortKeys.buf = rec->info[recInfo_sortKeys];
576 extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
577 extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
580 /* record going to be deleted */
581 if (!delkeys.buf_used)
583 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
584 zh->m_record_type, fname, recordOffset);
585 yaz_log (YLOG_WARN, "cannot delete file above, storeKeys false");
589 if (zh->records_processed < zh->m_file_verbose_limit)
590 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
591 zh->m_record_type, fname, recordOffset);
592 zh->records_deleted++;
594 dict_delete (zh->reg->matchDict, matchStr);
595 rec_del (zh->reg->records, &rec);
603 /* record going to be updated */
604 if (!delkeys.buf_used)
606 yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T,
607 zh->m_record_type, fname, recordOffset);
608 yaz_log (YLOG_WARN, "cannot update file above, storeKeys false");
612 if (zh->records_processed < zh->m_file_verbose_limit)
613 yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T,
614 zh->m_record_type, fname, recordOffset);
615 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
616 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
617 zh->records_updated++;
621 /* update file type */
622 xfree (rec->info[recInfo_fileType]);
623 rec->info[recInfo_fileType] =
624 rec_strdup (zh->m_record_type, &rec->size[recInfo_fileType]);
626 /* update filename */
627 xfree (rec->info[recInfo_filename]);
628 rec->info[recInfo_filename] =
629 rec_strdup (fname, &rec->size[recInfo_filename]);
631 /* update delete keys */
632 xfree (rec->info[recInfo_delKeys]);
633 if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1)
635 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
636 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
637 zh->reg->keys.buf = NULL;
638 zh->reg->keys.buf_max = 0;
642 rec->info[recInfo_delKeys] = NULL;
643 rec->size[recInfo_delKeys] = 0;
646 /* update sort keys */
647 xfree (rec->info[recInfo_sortKeys]);
649 rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
650 rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
651 zh->reg->sortKeys.buf = NULL;
652 zh->reg->sortKeys.buf_max = 0;
654 /* save file size of original record */
655 zebraExplain_recordBytesIncrement (zh->reg->zei,
656 - recordAttr->recordSize);
657 recordAttr->recordSize = fi->file_moffset - recordOffset;
658 if (!recordAttr->recordSize)
659 recordAttr->recordSize = fi->file_max - recordOffset;
660 zebraExplain_recordBytesIncrement (zh->reg->zei,
661 recordAttr->recordSize);
663 /* set run-number for this record */
664 recordAttr->runNumber = zebraExplain_runNumberIncrement (zh->reg->zei,
667 /* update store data */
668 xfree (rec->info[recInfo_storeData]);
669 if (zh->store_data_buf)
671 rec->size[recInfo_storeData] = zh->store_data_size;
672 rec->info[recInfo_storeData] = zh->store_data_buf;
673 zh->store_data_buf = 0;
675 else if (zh->m_store_data)
677 rec->size[recInfo_storeData] = recordAttr->recordSize;
678 rec->info[recInfo_storeData] = (char *)
679 xmalloc (recordAttr->recordSize);
680 if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
682 yaz_log (YLOG_ERRNO|YLOG_FATAL, "seek to " PRINTF_OFF_T " in %s",
683 recordOffset, fname);
686 if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize)
687 < recordAttr->recordSize)
689 yaz_log (YLOG_ERRNO|YLOG_FATAL, "read %d bytes of %s",
690 recordAttr->recordSize, fname);
696 rec->info[recInfo_storeData] = NULL;
697 rec->size[recInfo_storeData] = 0;
699 /* update database name */
700 xfree (rec->info[recInfo_databaseName]);
701 rec->info[recInfo_databaseName] =
702 rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
705 recordAttr->recordOffset = recordOffset;
707 /* commit this record */
708 rec_put (zh->reg->records, &rec);
713 int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
720 struct file_read_info *fi;
721 const char *original_record_type = 0;
723 void *recTypeClientData;
725 if (!zh->m_group || !*zh->m_group)
728 sprintf (gprefix, "%s.", zh->m_group);
730 yaz_log (YLOG_DEBUG, "fileExtract %s", fname);
732 /* determine file extension */
734 for (i = strlen(fname); --i >= 0; )
737 else if (fname[i] == '.')
739 strcpy (ext, fname+i+1);
742 /* determine file type - depending on extension */
743 original_record_type = zh->m_record_type;
744 if (!zh->m_record_type)
746 sprintf (ext_res, "%srecordType.%s", gprefix, ext);
747 zh->m_record_type = res_get (zh->res, ext_res);
749 if (!zh->m_record_type)
751 if (zh->records_processed < zh->m_file_verbose_limit)
752 yaz_log (YLOG_LOG, "? %s", fname);
755 /* determine match criteria */
756 if (!zh->m_record_id)
758 sprintf (ext_res, "%srecordId.%s", gprefix, ext);
759 zh->m_record_id = res_get (zh->res, ext_res);
763 recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
764 &recTypeClientData)))
766 yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
770 switch(recType->version)
775 yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
777 if (sysno && deleteFlag)
783 if (zh->path_reg && !yaz_is_abspath (fname))
785 strcpy (full_rep, zh->path_reg);
786 strcat (full_rep, "/");
787 strcat (full_rep, fname);
790 strcpy (full_rep, fname);
793 if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
795 yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
796 zh->m_record_type = original_record_type;
800 fi = file_read_start (fd);
803 fi->file_moffset = fi->file_offset;
804 fi->file_more = 0; /* file_end not called (yet) */
805 r = file_extract_record (zh, sysno, fname, deleteFlag, fi, 1,
806 recType, recTypeClientData);
808 { /* file_end has been called so reset offset .. */
809 fi->file_offset = fi->file_moffset;
810 lseek(fi->fd, fi->file_moffset, SEEK_SET);
817 zh->m_record_type = original_record_type;
822 If sysno is provided, then it's used to identify the reocord.
823 If not, and match_criteria is provided, then sysno is guessed
824 If not, and a record is provided, then sysno is got from there
827 ZEBRA_RES buffer_extract_record (ZebraHandle zh,
828 const char *buf, size_t buf_size,
831 const char *recordType,
833 const char *match_criteria,
838 RecordAttr *recordAttr;
839 struct recExtractCtrl extractCtrl;
841 const char *matchStr = 0;
842 RecType recType = NULL;
845 long recordOffset = 0;
846 struct zebra_fetch_control fc;
847 const char *pr_fname = fname; /* filename to print .. */
848 int show_progress = zh->records_processed < zh->m_file_verbose_limit ? 1:0;
851 pr_fname = "<no file>"; /* make it printable if file is omitted */
854 fc.record_int_buf = buf;
855 fc.record_int_len = buf_size;
856 fc.record_int_pos = 0;
858 fc.record_offset = 0;
860 extractCtrl.offset = 0;
861 extractCtrl.readf = zebra_record_int_read;
862 extractCtrl.seekf = zebra_record_int_seek;
863 extractCtrl.tellf = zebra_record_int_tell;
864 extractCtrl.endf = zebra_record_int_end;
865 extractCtrl.first_record = 1;
866 extractCtrl.fh = &fc;
868 create_rec_keys_codec(&zh->reg->keys);
870 zh->reg->sortKeys.buf_used = 0;
872 if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
874 if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
875 zh->m_explain_database))
879 if (recordType && *recordType)
881 yaz_log (YLOG_DEBUG, "Record type explicitly specified: %s", recordType);
882 recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
887 if (!(zh->m_record_type))
889 yaz_log (YLOG_WARN, "No such record type defined");
892 yaz_log (YLOG_DEBUG, "Get record type from rgroup: %s",zh->m_record_type);
893 recType = recType_byName (zh->reg->recTypes, zh->res,
894 zh->m_record_type, &clientData);
895 recordType = zh->m_record_type;
900 yaz_log (YLOG_WARN, "No such record type: %s", zh->m_record_type);
904 extractCtrl.init = extract_init;
905 extractCtrl.tokenAdd = extract_token_add;
906 extractCtrl.schemaAdd = extract_schema_add;
907 extractCtrl.dh = zh->reg->dh;
908 extractCtrl.handle = zh;
909 extractCtrl.match_criteria[0] = '\0';
911 init_extractCtrl(zh, &extractCtrl);
913 extract_set_store_data_prepare(&extractCtrl);
915 r = (*recType->extract)(clientData, &extractCtrl);
917 if (r == RECCTRL_EXTRACT_EOF)
919 else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
921 /* error occured during extraction ... */
922 yaz_log (YLOG_WARN, "extract error: generic");
925 else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
927 /* error occured during extraction ... */
928 yaz_log (YLOG_WARN, "extract error: no such filter");
934 if (extractCtrl.match_criteria[0])
935 match_criteria = extractCtrl.match_criteria;
939 if (match_criteria && *match_criteria) {
940 matchStr = match_criteria;
942 if (zh->m_record_id && *zh->m_record_id) {
943 matchStr = fileMatchStr (zh, &zh->reg->keys, pr_fname,
947 yaz_log (YLOG_WARN, "Bad match criteria (recordID)");
953 rinfo = dict_lookup (zh->reg->matchDict, matchStr);
956 assert(*rinfo == sizeof(*sysno));
957 memcpy (sysno, rinfo+1, sizeof(*sysno));
961 if (zh->reg->keys.buf_used == 0)
963 /* the extraction process returned no information - the record
964 is probably empty - unless flagShowRecords is in use */
975 yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
976 pr_fname, (long) recordOffset);
977 yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
981 yaz_log (YLOG_LOG, "add %s %s %ld", recordType, pr_fname,
982 (long) recordOffset);
983 rec = rec_new (zh->reg->records);
987 recordAttr = rec_init_attr (zh->reg->zei, rec);
991 dict_insert (zh->reg->matchDict, matchStr,
992 sizeof(*sysno), sysno);
994 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
995 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
997 zh->records_inserted++;
1001 /* record already exists */
1002 struct recKeys delkeys;
1003 struct sortKeys sortKeys;
1008 yaz_log (YLOG_LOG, "skipped %s %s %ld",
1009 recordType, pr_fname, (long) recordOffset);
1014 rec = rec_get (zh->reg->records, *sysno);
1017 recordAttr = rec_init_attr (zh->reg->zei, rec);
1019 if (!force_update) {
1020 if (recordAttr->runNumber ==
1021 zebraExplain_runNumberIncrement (zh->reg->zei, 0))
1024 yaz_log (YLOG_LOG, "skipped %s %s %ld", recordType,
1025 pr_fname, (long) recordOffset);
1026 extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
1033 delkeys.buf_used = rec->size[recInfo_delKeys];
1034 delkeys.buf = rec->info[recInfo_delKeys];
1036 sortKeys.buf_used = rec->size[recInfo_sortKeys];
1037 sortKeys.buf = rec->info[recInfo_sortKeys];
1039 extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
1040 extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
1043 /* record going to be deleted */
1044 if (!delkeys.buf_used)
1048 yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
1049 pr_fname, (long) recordOffset);
1050 yaz_log (YLOG_WARN, "cannot delete file above, "
1057 yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
1058 pr_fname, (long) recordOffset);
1059 zh->records_deleted++;
1061 dict_delete (zh->reg->matchDict, matchStr);
1062 rec_del (zh->reg->records, &rec);
1070 /* record going to be updated */
1071 if (!delkeys.buf_used)
1075 yaz_log (YLOG_LOG, "update %s %s %ld", recordType,
1076 pr_fname, (long) recordOffset);
1077 yaz_log (YLOG_WARN, "cannot update file above, storeKeys false");
1083 yaz_log (YLOG_LOG, "update %s %s %ld", recordType,
1084 pr_fname, (long) recordOffset);
1085 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
1086 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
1087 zh->records_updated++;
1091 /* update file type */
1092 xfree (rec->info[recInfo_fileType]);
1093 rec->info[recInfo_fileType] =
1094 rec_strdup (recordType, &rec->size[recInfo_fileType]);
1096 /* update filename */
1097 xfree (rec->info[recInfo_filename]);
1098 rec->info[recInfo_filename] =
1099 rec_strdup (fname, &rec->size[recInfo_filename]);
1101 /* update delete keys */
1102 xfree (rec->info[recInfo_delKeys]);
1103 if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1)
1105 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
1106 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
1107 zh->reg->keys.buf = NULL;
1108 zh->reg->keys.buf_max = 0;
1112 rec->info[recInfo_delKeys] = NULL;
1113 rec->size[recInfo_delKeys] = 0;
1116 /* update sort keys */
1117 xfree (rec->info[recInfo_sortKeys]);
1119 rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
1120 rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
1121 zh->reg->sortKeys.buf = NULL;
1122 zh->reg->sortKeys.buf_max = 0;
1124 /* save file size of original record */
1125 zebraExplain_recordBytesIncrement (zh->reg->zei,
1126 - recordAttr->recordSize);
1128 recordAttr->recordSize = fi->file_moffset - recordOffset;
1129 if (!recordAttr->recordSize)
1130 recordAttr->recordSize = fi->file_max - recordOffset;
1132 recordAttr->recordSize = buf_size;
1134 zebraExplain_recordBytesIncrement (zh->reg->zei,
1135 recordAttr->recordSize);
1137 /* set run-number for this record */
1138 recordAttr->runNumber =
1139 zebraExplain_runNumberIncrement (zh->reg->zei, 0);
1141 /* update store data */
1142 xfree (rec->info[recInfo_storeData]);
1144 /* update store data */
1145 if (zh->store_data_buf)
1147 rec->size[recInfo_storeData] = zh->store_data_size;
1148 rec->info[recInfo_storeData] = zh->store_data_buf;
1149 zh->store_data_buf = 0;
1151 else if (zh->m_store_data)
1153 rec->size[recInfo_storeData] = recordAttr->recordSize;
1154 rec->info[recInfo_storeData] = (char *)
1155 xmalloc (recordAttr->recordSize);
1156 memcpy (rec->info[recInfo_storeData], buf, recordAttr->recordSize);
1160 rec->info[recInfo_storeData] = NULL;
1161 rec->size[recInfo_storeData] = 0;
1163 /* update database name */
1164 xfree (rec->info[recInfo_databaseName]);
1165 rec->info[recInfo_databaseName] =
1166 rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
1169 recordAttr->recordOffset = recordOffset;
1171 /* commit this record */
1172 rec_put (zh->reg->records, &rec);
1177 int explain_extract (void *handle, Record rec, data1_node *n)
1179 ZebraHandle zh = (ZebraHandle) handle;
1180 struct recExtractCtrl extractCtrl;
1182 if (zebraExplain_curDatabase (zh->reg->zei,
1183 rec->info[recInfo_databaseName]))
1186 if (zebraExplain_newDatabase (zh->reg->zei,
1187 rec->info[recInfo_databaseName], 0))
1191 create_rec_keys_codec(&zh->reg->keys);
1193 zh->reg->sortKeys.buf_used = 0;
1195 extractCtrl.init = extract_init;
1196 extractCtrl.tokenAdd = extract_token_add;
1197 extractCtrl.schemaAdd = extract_schema_add;
1198 extractCtrl.dh = zh->reg->dh;
1200 init_extractCtrl(zh, &extractCtrl);
1202 extractCtrl.flagShowRecords = 0;
1203 extractCtrl.match_criteria[0] = '\0';
1204 extractCtrl.handle = handle;
1205 extractCtrl.first_record = 1;
1207 extract_set_store_data_prepare(&extractCtrl);
1210 grs_extract_tree(&extractCtrl, n);
1212 if (rec->size[recInfo_delKeys])
1214 struct recKeys delkeys;
1215 struct sortKeys sortkeys;
1217 delkeys.buf_used = rec->size[recInfo_delKeys];
1218 delkeys.buf = rec->info[recInfo_delKeys];
1220 sortkeys.buf_used = rec->size[recInfo_sortKeys];
1221 sortkeys.buf = rec->info[recInfo_sortKeys];
1223 extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys);
1224 extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys);
1226 extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys);
1227 extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys);
1229 xfree (rec->info[recInfo_delKeys]);
1230 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
1231 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
1232 zh->reg->keys.buf = NULL;
1233 zh->reg->keys.buf_max = 0;
1235 xfree (rec->info[recInfo_sortKeys]);
1236 rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
1237 rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
1238 zh->reg->sortKeys.buf = NULL;
1239 zh->reg->sortKeys.buf_max = 0;
1244 void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
1245 int cmd, struct recKeys *reckeys)
1247 void *decode_handle = iscz1_start();
1250 ZebraExplainInfo zei = zh->reg->zei;
1252 if (!zh->reg->key_buf)
1254 int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8"));
1257 yaz_log(YLOG_WARN, "Invalid memory setting, using default 8 MB");
1260 /* FIXME: That "8" should be in a default settings include */
1261 /* not hard-coded here! -H */
1262 zh->reg->key_buf = (char**) xmalloc (mem);
1263 zh->reg->ptr_top = mem/sizeof(char*);
1265 zh->reg->key_buf_used = 0;
1266 zh->reg->key_file_no = 0;
1268 zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1270 while (off < reckeys->buf_used)
1272 const char *src = reckeys->buf + off;
1274 char *dst = (char*) &key;
1276 iscz1_decode(decode_handle, &dst, &src);
1277 assert(key.len == 4);
1279 if (zh->reg->key_buf_used + 1024 >
1280 (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*))
1281 extract_flushWriteKeys (zh,0);
1283 assert(zh->reg->ptr_i > 0);
1284 (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] =
1285 (char*)zh->reg->key_buf + zh->reg->key_buf_used;
1287 ch = (int) key.mem[0]; /* ordinal for field/use/attribute */
1289 zh->reg->key_buf_used +=
1290 key_SU_encode (ch,((char*)zh->reg->key_buf) +
1291 zh->reg->key_buf_used);
1293 ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++;
1295 ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = '\0';
1296 ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = cmd;
1299 if (key.mem[1]) /* filter specified record ID */
1300 key.mem[0] = key.mem[1];
1303 key.mem[1] = key.mem[2]; /* section_id */
1304 key.mem[2] = key.mem[3]; /* sequence .. */
1306 memcpy ((char*)zh->reg->key_buf + zh->reg->key_buf_used,
1308 (zh->reg->key_buf_used) += sizeof(key);
1309 off = src - reckeys->buf;
1311 assert (off == reckeys->buf_used);
1312 iscz1_stop(decode_handle);
1315 void extract_flushWriteKeys (ZebraHandle zh, int final)
1316 /* optimizing: if final=1, and no files written yet */
1317 /* push the keys directly to merge, sidestepping the */
1318 /* temp file altogether. Speeds small updates */
1321 char out_fname[200];
1323 struct encode_info encode_info;
1324 int ptr_i = zh->reg->ptr_i;
1329 if (!zh->reg->key_buf || ptr_i <= 0)
1331 yaz_log (YLOG_DEBUG, " nothing to flush section=%d buf=%p i=%d",
1332 zh->reg->key_file_no, zh->reg->key_buf, ptr_i);
1333 yaz_log (YLOG_DEBUG, " buf=%p ",
1335 yaz_log (YLOG_DEBUG, " ptr=%d ",zh->reg->ptr_i);
1336 yaz_log (YLOG_DEBUG, " reg=%p ",zh->reg);
1341 (zh->reg->key_file_no)++;
1342 yaz_log (YLOG_LOG, "sorting section %d", (zh->reg->key_file_no));
1343 yaz_log (YLOG_DEBUG, " sort_buff at %p n=%d",
1344 zh->reg->key_buf + zh->reg->ptr_top - ptr_i,ptr_i);
1346 qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i,
1347 sizeof(char*), key_qsort_compare);
1349 /* zebra.cfg: tempfiles:
1350 Y: always use temp files (old way)
1351 A: use temp files, if more than one (auto)
1352 = if this is both the last and the first
1353 N: never bother with temp files (new) */
1355 temp_policy=toupper(res_get_def(zh->res,"tempfiles","auto")[0]);
1356 if (temp_policy != 'Y' && temp_policy != 'N' && temp_policy != 'A') {
1357 yaz_log (YLOG_WARN, "Illegal tempfiles setting '%c'. using 'Auto' ",
1362 if ( ( temp_policy =='N' ) || /* always from memory */
1363 ( ( temp_policy =='A' ) && /* automatic */
1364 (zh->reg->key_file_no == 1) && /* this is first time */
1365 (final) ) ) /* and last (=only) time */
1366 { /* go directly from memory */
1367 zh->reg->key_file_no =0; /* signal not to read files */
1368 zebra_index_merge(zh);
1370 zh->reg->key_buf_used = 0;
1374 /* Not doing directly from memory, write into a temp file */
1375 extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no);
1377 if (!(outf = fopen (out_fname, "wb")))
1379 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname);
1382 yaz_log (YLOG_LOG, "writing section %d", zh->reg->key_file_no);
1383 prevcp = cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i];
1385 encode_key_init (&encode_info);
1386 encode_key_write (cp, &encode_info, outf);
1390 cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i];
1391 if (strcmp (cp, prevcp))
1393 encode_key_flush ( &encode_info, outf);
1394 encode_key_init (&encode_info);
1395 encode_key_write (cp, &encode_info, outf);
1399 encode_key_write (cp + strlen(cp), &encode_info, outf);
1401 encode_key_flush ( &encode_info, outf);
1403 qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_x_compare);
1404 extract_get_fname_tmp (out_fname, key_file_no);
1406 if (!(outf = fopen (out_fname, "wb")))
1408 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname);
1411 yaz_log (YLOG_LOG, "writing section %d", key_file_no);
1413 prevcp = key_buf[ptr_top-i];
1415 if (!--i || strcmp (prevcp, key_buf[ptr_top-i]))
1417 key_y_len = strlen(prevcp)+1;
1419 yaz_log (YLOG_LOG, "key_y_len: %2d %02x %02x %s",
1420 key_y_len, prevcp[0], prevcp[1], 2+prevcp);
1422 qsort (key_buf + ptr_top-ptr_i, ptr_i - i,
1423 sizeof(char*), key_y_compare);
1424 cp = key_buf[ptr_top-ptr_i];
1426 encode_key_init (&encode_info);
1427 encode_key_write (cp, &encode_info, outf);
1430 cp = key_buf[ptr_top-ptr_i];
1431 encode_key_write (cp+key_y_len, &encode_info, outf);
1433 encode_key_flush ( &encode_info, outf);
1436 prevcp = key_buf[ptr_top-ptr_i];
1441 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname);
1444 yaz_log (YLOG_LOG, "finished section %d", zh->reg->key_file_no);
1446 zh->reg->key_buf_used = 0;
1449 void extract_add_it_key (ZebraHandle zh,
1451 const char *str, int slen, struct it_key *key)
1454 struct recKeys *keys = &zh->reg->keys;
1455 const char *src = (char*) key;
1457 if (keys->buf_used+1024 > keys->buf_max)
1459 char *b = (char *) xmalloc (keys->buf_max += 128000);
1460 if (keys->buf_used > 0)
1461 memcpy (b, keys->buf, keys->buf_used);
1465 dst = keys->buf + keys->buf_used;
1467 iscz1_encode(keys->codec_handle, &dst, &src);
1470 memcpy (dst, str, slen);
1473 keys->buf_used = dst - keys->buf;
1476 void extract_add_index_string (RecWord *p, const char *str, int length)
1480 ZebraHandle zh = p->extractCtrl->handle;
1481 ZebraExplainInfo zei = zh->reg->zei;
1486 ch = zebraExplain_lookup_attr_str(zei, p->attrStr);
1488 ch = zebraExplain_add_attr_str(zei, p->attrStr);
1492 ch = zebraExplain_lookup_attr_su(zei, p->attrSet, p->attrUse);
1494 ch = zebraExplain_add_attr_su(zei, p->attrSet, p->attrUse);
1498 key.mem[1] = p->record_id;
1499 key.mem[2] = p->section_id;
1500 key.mem[3] = p->seqno;
1503 /* just for debugging .. */
1504 yaz_log(YLOG_LOG, "add: set=%d use=%d "
1505 "record_id=%lld section_id=%lld seqno=%lld",
1506 p->attrSet, p->attrUse, p->record_id, p->section_id, p->seqno);
1509 extract_add_it_key(p->extractCtrl->handle, p->reg_type, str,
1513 static void extract_add_sort_string (RecWord *p, const char *str,
1516 ZebraHandle zh = p->extractCtrl->handle;
1517 struct sortKeys *sk = &zh->reg->sortKeys;
1520 while (off < sk->buf_used)
1524 off += key_SU_decode(&set, sk->buf + off);
1525 off += key_SU_decode(&use, sk->buf + off);
1526 off += key_SU_decode(&slen, sk->buf + off);
1528 if (p->attrSet == set && p->attrUse == use)
1531 assert (off == sk->buf_used);
1533 if (sk->buf_used + IT_MAX_WORD > sk->buf_max)
1537 b = (char *) xmalloc (sk->buf_max += 128000);
1538 if (sk->buf_used > 0)
1539 memcpy (b, sk->buf, sk->buf_used);
1543 off += key_SU_encode(p->attrSet, sk->buf + off);
1544 off += key_SU_encode(p->attrUse, sk->buf + off);
1545 off += key_SU_encode(length, sk->buf + off);
1546 memcpy (sk->buf + off, str, length);
1547 sk->buf_used = off + length;
1550 void extract_add_string (RecWord *p, const char *string, int length)
1552 assert (length > 0);
1553 if (zebra_maps_is_sort (p->zebra_maps, p->reg_type))
1554 extract_add_sort_string (p, string, length);
1556 extract_add_index_string (p, string, length);
1559 static void extract_add_incomplete_field (RecWord *p)
1561 const char *b = p->term_buf;
1562 int remain = p->term_len;
1563 const char **map = 0;
1565 yaz_log(YLOG_DEBUG, "Incomplete field, w='%.*s'", p->term_len, p->term_buf);
1568 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0);
1572 char buf[IT_MAX_WORD+1];
1576 while (map && *map && **map == *CHR_SPACE)
1578 remain = p->term_len - (b - p->term_buf);
1580 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0);
1587 while (map && *map && **map != *CHR_SPACE)
1589 const char *cp = *map;
1591 while (i < IT_MAX_WORD && *cp)
1593 remain = p->term_len - (b - p->term_buf);
1595 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0);
1601 extract_add_string (p, buf, i);
1606 static void extract_add_complete_field (RecWord *p)
1608 const char *b = p->term_buf;
1609 char buf[IT_MAX_WORD+1];
1610 const char **map = 0;
1611 int i = 0, remain = p->term_len;
1613 yaz_log(YLOG_DEBUG, "Complete field, w='%.*s'",
1614 p->term_len, p->term_buf);
1617 map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain, 1);
1619 while (remain > 0 && i < IT_MAX_WORD)
1621 while (map && *map && **map == *CHR_SPACE)
1623 remain = p->term_len - (b - p->term_buf);
1627 int first = i ? 0 : 1; /* first position */
1628 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, first);
1636 if (i && i < IT_MAX_WORD)
1637 buf[i++] = *CHR_SPACE;
1638 while (map && *map && **map != *CHR_SPACE)
1640 const char *cp = *map;
1642 if (**map == *CHR_CUT)
1648 if (i >= IT_MAX_WORD)
1650 yaz_log(YLOG_DEBUG, "Adding string to index '%d'", **map);
1651 while (i < IT_MAX_WORD && *cp)
1654 remain = p->term_len - (b - p->term_buf);
1657 map = zebra_maps_input (p->zebra_maps, p->reg_type, &b,
1666 extract_add_string (p, buf, i);
1669 void extract_token_add (RecWord *p)
1673 yaz_log (YLOG_LOG, "token_add "
1674 "reg_type=%c attrSet=%d attrUse=%d seqno=%d s=%.*s",
1675 p->reg_type, p->attrSet, p->attrUse, p->seqno, p->length,
1678 if ((wrbuf = zebra_replace(p->zebra_maps, p->reg_type, 0,
1679 p->term_buf, p->term_len)))
1681 p->term_buf = wrbuf_buf(wrbuf);
1682 p->term_len = wrbuf_len(wrbuf);
1684 if (zebra_maps_is_complete (p->zebra_maps, p->reg_type))
1685 extract_add_complete_field (p);
1687 extract_add_incomplete_field(p);
1690 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1691 void *buf, size_t sz)
1693 ZebraHandle zh = (ZebraHandle) p->handle;
1695 xfree(zh->store_data_buf);
1696 zh->store_data_buf = 0;
1697 zh->store_data_size = 0;
1700 zh->store_data_buf = xmalloc(sz);
1701 zh->store_data_size = sz;
1702 memcpy(zh->store_data_buf, buf, sz);
1706 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1708 ZebraHandle zh = (ZebraHandle) p->handle;
1709 xfree(zh->store_data_buf);
1710 zh->store_data_buf = 0;
1711 zh->store_data_size = 0;
1712 p->setStoreData = extract_set_store_data_cb;
1715 void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid)
1717 ZebraHandle zh = (ZebraHandle) p->handle;
1718 zebraExplain_addSchema (zh->reg->zei, oid);
1721 void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
1722 int cmd, struct sortKeys *sk)
1724 SortIdx sortIdx = zh->reg->sortIdx;
1727 sortIdx_sysno (sortIdx, sysno);
1729 while (off < sk->buf_used)
1733 off += key_SU_decode(&set, sk->buf + off);
1734 off += key_SU_decode(&use, sk->buf + off);
1735 off += key_SU_decode(&slen, sk->buf + off);
1737 sortIdx_type(sortIdx, use);
1739 sortIdx_add(sortIdx, sk->buf + off, slen);
1741 sortIdx_add(sortIdx, "", 1);
1746 void encode_key_init (struct encode_info *i)
1755 i->encode_handle = iscz1_start();
1761 /* this is the old encode_key_write
1762 * may be deleted once we are confident that the new works
1765 void encode_key_write (char *k, struct encode_info *i, FILE *outf)
1768 char *bp = i->buf, *bp0;
1769 const char *src = (char *) &key;
1771 /* copy term to output buf */
1772 while ((*bp++ = *k++))
1774 /* and copy & align key so we can mangle */
1775 memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */
1778 iscz1_encode(i->encode_handle, &bp, &src);
1779 *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */
1780 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1782 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
1787 void encode_key_flush (struct encode_info *i, FILE *outf)
1788 { /* dummy routine */
1789 iscz1_stop(i->encode_handle);
1794 /* new encode_key_write
1795 * The idea is to buffer one more key, and compare them
1796 * If we are going to delete and insert the same key,
1797 * we may as well not bother. Should make a difference in
1798 * updates with small modifications (appending to a mbox)
1800 void encode_key_write (char *k, struct encode_info *i, FILE *outf)
1805 if (*k) /* first time for new key */
1808 while ((*bp++ = *k++))
1810 i->keylen= bp - i->buf -1;
1811 assert(i->keylen+1+sizeof(struct it_key) < ENCODE_BUFLEN);
1815 bp=i->buf + i->keylen;
1820 memcpy (&key, k+1, sizeof(struct it_key));
1821 if (0==i->prevsys) /* no previous filter, fill up */
1823 i->prevsys=key.sysno;
1824 i->prevseq=key.seqno;
1827 else if ( (i->prevsys==key.sysno) &&
1828 (i->prevseq==key.seqno) &&
1830 { /* same numbers, diff cmd, they cancel out */
1834 { /* different stuff, write previous, move buf */
1835 bp = encode_key_int ( (i->prevsys - i->sysno) * 2 + i->prevcmd, bp);
1836 if (i->sysno != i->prevsys)
1838 i->sysno = i->prevsys;
1841 else if (!i->seqno && !i->prevseq && i->cmd == i->prevcmd)
1843 return; /* ??? Filters some sort of duplicates away */
1844 /* ??? Can this ever happen -H 15oct02 */
1846 bp = encode_key_int (i->prevseq - i->seqno, bp);
1847 i->seqno = i->prevseq;
1848 i->cmd = i->prevcmd;
1849 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1851 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
1854 i->keylen=0; /* ok, it's written, forget it */
1855 i->prevsys=key.sysno;
1856 i->prevseq=key.seqno;
1861 void encode_key_flush (struct encode_info *i, FILE *outf)
1862 { /* flush the last key from i */
1863 char *bp =i->buf + i->keylen;
1866 return; /* nothing to flush */
1869 bp = encode_key_int ( (i->prevsys - i->sysno) * 2 + i->prevcmd, bp);
1870 if (i->sysno != i->prevsys)
1872 i->sysno = i->prevsys;
1875 else if (!i->seqno && !i->prevseq && i->cmd == i->prevcmd)
1877 return; /* ??? Filters some sort of duplicates away */
1878 /* ??? Can this ever happen -H 15oct02 */
1880 bp = encode_key_int (i->prevseq - i->seqno, bp);
1881 i->seqno = i->prevseq;
1882 i->cmd = i->prevcmd;
1883 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1885 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
1888 i->keylen=0; /* ok, it's written, forget it */
1889 i->prevsys=0; /* forget the values too */