1 /* $Id: csvread.c,v 1.3 2005-12-06 15:36:38 adam Exp $
2 Copyright (C) 1995-2005
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
27 #include <yaz/yaz-util.h>
29 /* #include <d1_absyn.h> */
30 #include <idzebra/data1.h>
31 #include <idzebra/recgrs.h>
33 /* #include <assert.h> */
37 struct csv_getc_info {
44 int (*readf)(void *, char *, size_t);
64 /* char *field_names; */
69 static void grs_config_csv(void *clientData, Res res, const char *args)
72 struct csv_t *csvp = (struct csv_t*) clientData;
74 yaz_log (YLOG_LOG, "Called CSV filter grs_config_csv");
75 yaz_log (YLOG_LOG, "'%s'", args);
78 csvp->buf = nmem_malloc(csvp->nmem, csvp->buf_size);
79 csvp->name_size = 256;
80 csvp->value_size = 4096;
81 csvp->value = nmem_malloc(csvp->nmem, csvp->value_size);
83 csvp->field_char = '|';
84 csvp->record_char = '\n';
85 csvp->string_char = 0;
86 csvp->root_element = nmem_strdup(csvp->nmem, "csv");
89 csvp->max_nr_fields = 512;
91 /* csvp->field_names = 0; */ /*nmem_strdup(csvp->nmem, "a|b|c|d|e");*/
94 = nmem_malloc(csvp->nmem,
95 sizeof(*(csvp->field_name)) * csvp->max_nr_fields);
96 for (i = 0; i < csvp->max_nr_fields; i++){
97 csvp->field_name[i] = 0;
100 /* know field names from config file */
101 /*if (strlen(csvp->field_names))
102 yaz_log (YLOG_LOG, "CSV filter grs_config_csv field names");
105 yaz_log (YLOG_LOG, "Ended CSV filter grs_config_csv");
109 static data1_node *grs_read_csv (struct grs_read_info *gri)
111 data1_node *root_node = 0;
112 data1_node *node = 0;
113 struct csv_t *csvp = (struct csv_t *)gri->clientData;
115 int end_of_record = 0;
118 char *cb = csvp->buf;
119 char *cv = csvp->value;
121 yaz_log (YLOG_LOG, "Called CSV filter grs_read_csv");
123 /* if on start of first line, read header line for dynamic configure */
124 if(csvp->field_line && gri->offset == 0)
127 while (!end_of_record){
130 /* configure grs.csv filter with first line in file containing field
133 yaz_log (YLOG_LOG, "CSV filter grs_read_csv reading header line");
135 /* create new memory for fieldname and value */
136 if (old_nr_fields < csvp->nr_fields){
138 "CSV filter grs_read_csv name:'%d' ", csvp->nr_fields);
139 old_nr_fields = csvp->nr_fields;
140 csvp->field_name[csvp->nr_fields]
141 = nmem_malloc(csvp->nmem, csvp->name_size);
142 csvp->field_value[csvp->nr_fields]
143 = nmem_malloc(csvp->nmem, csvp->value_size);
145 /* read buf and copy values to field_name[] */
146 read_bytes = (*gri->readf)(gri->fh, csvp->buf, csvp->buf_size);
147 gri-> offset = (*gri->tellf)(gri->fh);
148 /* yaz_log(YLOG_LOG, "CSV filter grs_read_csv offset:'%d' ", offset); */
152 /* read buf and copy values to field_value[] */
153 read_bytes = (*gri->readf)(gri->fh, csvp->buf, csvp->buf_size);
154 gri->offset = (*gri->tellf)(gri->fh);
155 yaz_log(YLOG_LOG, "CSV filter grs_read_csv offset:'%d' ", offset);
161 /* read new buffer from file */
162 read_bytes = (*gri->readf)(gri->fh, csvp->buf, csvp->buf_size);
164 yaz_log (YLOG_LOG, "CSV filter grs_read_csv read_bytes %d", read_bytes);
165 yaz_log (YLOG_LOG, "CSV filter grs_read_csv csvp->buf %s", csvp->buf);
167 gri->offset = (*gri->tellf)(gri->fh);
168 yaz_log(YLOG_LOG, "CSV filter grs_read_csv gri->offset:'%d' ",
173 while ((cb - csvp->buf < read_bytes)
174 && (cv - csvp->value < csvp->value_size)
177 if (*cb == csvp->field_char){
178 /* if field finished */
181 /* read field names from header line */
182 if (csvp->nr_fields < csvp->max_nr_fields){
183 csvp->field_name[csvp->nr_fields]
184 = nmem_strdup(csvp->nmem, csvp->value);
187 yaz_log (YLOG_LOG, "CSV filter grs_read_csv field %d name '%s'",
188 field_nr, csvp->value);
190 yaz_log (YLOG_WARN, "CSV filter grs_read_csv field %d name '%s' "
191 "exceeds configured max number of fields %d",
192 field_nr, csvp->value, csvp->max_nr_fields);
195 /* process following value line fields */
196 if (field_nr < csvp->nr_fields){
197 /* less or qual fields number */
198 yaz_log (YLOG_LOG, "CSV filter grs_read_csv field %d %s: '%s'",
199 field_nr, csvp->field_name[field_nr], csvp->value);
201 /* too many fields */
202 yaz_log (YLOG_WARN, "CSV filter grs_read_csv field value %d %s "
203 "exceeds dynamic configured number of fields %d",
204 field_nr, csvp->value, csvp->nr_fields);
208 /* advance buffer and proceed to next field */
212 } else if (*cb == csvp->record_char){
213 /* if record finished */
214 /* advance buffer and proceed to record */
221 yaz_log (YLOG_LOG, "CSV filter grs_read_csv header end");
224 yaz_log (YLOG_LOG, "CSV filter grs_read_csv record end");
227 /* just plain char to be stored in value, no special action at all */
228 if (csvp->lower_case && read_header){
240 (*gri->endf)(gri->fh, offset - 1); */
243 /* try to build GRS node and document */
245 root_node = data1_mk_root(gri->dh, gri->mem, csvp->root_element);
246 node = data1_mk_node2(gri->dh, gri->mem, DATA1N_data, root_node);
247 node = data1_mk_tag(gri->dh, gri->mem, "pr_name_gn", 0, node);
248 data1_mk_text_n(gri->dh, gri->mem, csvp->buf, read_bytes, node);
251 yaz_log (YLOG_WARN, "empty CSV record of type '%s' "
252 "near file offset %d "
253 "or missing abstract syntax file '%s.abs'",
254 csvp->root_element, (int)gri->offset, csvp->root_element);
258 yaz_log (YLOG_LOG, "Ended CSV filter grs_read_csv");
262 static void *grs_init_csv(Res res, RecType recType)
264 NMEM m = nmem_create();
265 struct csv_t *csvp = (struct csv_t *) nmem_malloc(m, sizeof(*csvp));
266 yaz_log (YLOG_LOG, "Called CSV filter grs_init_csv");
268 yaz_log (YLOG_LOG, "Ended CSV filter grs_init_csv");
272 static void grs_destroy_csv(void *clientData)
274 struct csv_t *csvp = (struct csv_t*) clientData;
276 yaz_log (YLOG_LOG, "Called CSV filter grs_destroy_csv");
278 nmem_destroy(csvp->nmem);
281 yaz_log (YLOG_LOG, "Ended CSV filter grs_destroy_csv");
284 static int grs_extract_csv(void *clientData, struct recExtractCtrl *ctrl)
287 /* struct csv_t *csvp = (struct csv_t*) clientData; */
289 yaz_log (YLOG_LOG, "Called CSV filter grs_extract_csv");
290 yaz_log (YLOG_LOG, "recExtractCtr fh %d", (int)ctrl->fh);
291 yaz_log (YLOG_LOG, "recExtractCtr offset %d", (int)ctrl->offset);
293 res = zebra_grs_extract(clientData, ctrl, grs_read_csv);
295 yaz_log (YLOG_LOG, "recExtractCtr fh %d", (int)ctrl->fh);
296 yaz_log (YLOG_LOG, "recExtractCtr offset %d", (int)ctrl->offset);
297 yaz_log (YLOG_LOG, "Ended CSV filter grs_extract_csv");
302 static int grs_retrieve_csv(void *clientData, struct recRetrieveCtrl *ctrl)
305 /* struct csv_t *csvp = (struct csv_t*) clientData; */
307 yaz_log (YLOG_LOG, "Called CSV filter grs_retrieve_csv");
308 res = zebra_grs_retrieve(clientData, ctrl, grs_read_csv);
309 yaz_log (YLOG_LOG, "Ended CSV filter grs_retrieve_csv");
314 static struct recType grs_type_csv =
326 #ifdef IDZEBRA_STATIC_GRS_CSV
327 idzebra_filter_grs_csv