1 /* $Id: charmap.c,v 1.28 2004-03-09 15:12:15 adam Exp $
2 Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
26 * Support module to handle character-conversions into and out of the
34 typedef unsigned ucs4_t;
36 #include <yaz/yaz-util.h>
40 #define CHR_MAXSTR 1024
41 #define CHR_MAXEQUIV 32
43 const char *CHR_UNKNOWN = "\001";
44 const char *CHR_SPACE = "\002";
45 const char *CHR_BASE = "\003";
49 chr_t_entry *input; /* mapping table for input data */
50 chr_t_entry *q_input; /* mapping table for queries */
51 unsigned char *output[256]; /* return mapping - for display of registers */
52 int base_uppercase; /* Start of upper-case ordinals */
57 * Character map trie node.
61 chr_t_entry **children; /* array of children */
62 unsigned char **target; /* target for this node, if any */
66 * General argument structure for callback functions (internal use only)
68 typedef struct chrwork
71 char string[CHR_MAXSTR+1];
75 * Add an entry to the character map.
77 static chr_t_entry *set_map_string(chr_t_entry *root, NMEM nmem,
78 const char *from, int len, char *to,
85 root = (chr_t_entry *) nmem_malloc(nmem, sizeof(*root));
91 if (!root->target || !root->target[0] || strcmp(root->target[0], to))
94 root->target && root->target[0] && root->target[0][0] &&
95 strcmp (root->target[0], CHR_UNKNOWN))
97 yaz_log (LOG_WARN, "duplicate entry for charmap from '%s'",
100 root->target = (unsigned char **)
101 nmem_malloc(nmem, sizeof(*root->target)*2);
102 root->target[0] = (unsigned char *) nmem_strdup(nmem, to);
112 root->children = (chr_t_entry **)
113 nmem_malloc(nmem, sizeof(chr_t_entry*) * 256);
114 for (i = 0; i < 256; i++)
115 root->children[i] = 0;
117 if (!(root->children[(unsigned char) *from] =
118 set_map_string(root->children[(unsigned char) *from], nmem,
119 from + 1, len - 1, to, from_0)))
125 static chr_t_entry *find_entry(chr_t_entry *t, const char **from, int len)
129 if (len && t->children && t->children[(unsigned char) **from])
131 const char *pos = *from;
134 if ((res = find_entry(t->children[(unsigned char) *pos],
140 /* no children match. use ourselves, if we have a target */
141 return t->target ? t : 0;
144 static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len)
149 { /* switch to next buffer */
155 if (*len > 0 && t->children && t->children[(unsigned char) **from])
157 const char *old_from = *from;
162 if ((res = find_entry_x(t->children[(unsigned char) *old_from],
169 /* no children match. use ourselves, if we have a target */
170 return t->target ? t : 0;
173 const char **chr_map_input_x(chrmaptab maptab, const char **from, int *len)
175 chr_t_entry *t = maptab->input;
178 if (!(res = find_entry_x(t, from, len)))
180 return (const char **) (res->target);
183 const char **chr_map_input(chrmaptab maptab, const char **from, int len)
185 chr_t_entry *t = maptab->input;
191 if (!(res = find_entry_x(t, from, len_tmp)))
193 return (const char **) (res->target);
196 const char *chr_map_output(chrmaptab maptab, const char **from, int len)
198 unsigned char c = ** (unsigned char **) from;
200 return (const char*) maptab->output[c];
203 unsigned char zebra_prim(char **s)
208 yaz_log (LOG_DEBUG, "prim %.3s", *s);
215 case '\\': c = '\\'; (*s)++; break;
216 case 'r': c = '\r'; (*s)++; break;
217 case 'n': c = '\n'; (*s)++; break;
218 case 't': c = '\t'; (*s)++; break;
219 case 's': c = ' '; (*s)++; break;
220 case 'x': sscanf(*s, "x%2x", &i); c = i; *s += 3; break;
231 sscanf(*s, "%3o", &i);
247 static int zebra_ucs4_strlen(ucs4_t *s)
255 ucs4_t zebra_prim_w(ucs4_t **s)
261 yaz_log (LOG_DEBUG, "prim %.3s", (char *) *s);
268 case '\\': c = '\\'; (*s)++; break;
269 case 'r': c = '\r'; (*s)++; break;
270 case 'n': c = '\n'; (*s)++; break;
271 case 't': c = '\t'; (*s)++; break;
272 case 's': c = ' '; (*s)++; break;
274 if (zebra_ucs4_strlen(*s) >= 3)
279 sscanf(fmtstr, "%x", &i);
294 if (zebra_ucs4_strlen(*s) >= 3)
300 sscanf(fmtstr, "%o", &i);
306 if (zebra_ucs4_strlen(*s) >= 5)
313 sscanf(fmtstr, "%x", &i);
327 yaz_log (LOG_DEBUG, "out %d", c);
333 * Add an entry to the value space.
335 static void fun_addentry(const char *s, void *data, int num)
337 chrmaptab tab = (chrmaptab) data;
340 tmp[0] = num; tmp[1] = '\0';
341 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s), tmp, 0);
342 tab->output[num + tab->base_uppercase] =
343 (unsigned char *) nmem_strdup(tab->nmem, s);
348 * Add a space-entry to the value space.
350 static void fun_addspace(const char *s, void *data, int num)
352 chrmaptab tab = (chrmaptab) data;
353 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s),
354 (char*) CHR_SPACE, 0);
358 * Create a string containing the mapped characters provided.
360 static void fun_mkstring(const char *s, void *data, int num)
362 chrwork *arg = (chrwork *) data;
363 const char **res, *p = s;
365 res = chr_map_input(arg->map, &s, strlen(s));
366 if (*res == (char*) CHR_UNKNOWN)
367 logf(LOG_WARN, "Map: '%s' has no mapping", p);
368 strncat(arg->string, *res, CHR_MAXSTR - strlen(arg->string));
369 arg->string[CHR_MAXSTR] = '\0';
373 * Add a map to the string contained in the argument.
375 static void fun_add_map(const char *s, void *data, int num)
377 chrwork *arg = (chrwork *) data;
379 assert(arg->map->input);
380 logf (LOG_DEBUG, "set map %.*s", (int) strlen(s), s);
381 set_map_string(arg->map->input, arg->map->nmem, s, strlen(s), arg->string,
383 for (s = arg->string; *s; s++)
384 logf (LOG_DEBUG, " %3d", (unsigned char) *s);
388 * Add a query map to the string contained in the argument.
390 static void fun_add_qmap(const char *s, void *data, int num)
392 chrwork *arg = (chrwork *) data;
394 assert(arg->map->q_input);
395 logf (LOG_DEBUG, "set qmap %.*s", (int) strlen(s), s);
396 set_map_string(arg->map->q_input, arg->map->nmem, s,
397 strlen(s), arg->string, 0);
398 for (s = arg->string; *s; s++)
399 logf (LOG_DEBUG, " %3d", (unsigned char) *s);
402 static int scan_to_utf8 (yaz_iconv_t t, ucs4_t *from, size_t inlen,
403 char *outbuf, size_t outbytesleft)
405 size_t inbytesleft = inlen * sizeof(ucs4_t);
406 char *inbuf = (char*) from;
410 *outbuf++ = *from; /* ISO-8859-1 is OK here */
413 ret = yaz_iconv (t, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
414 if (ret == (size_t) (-1))
416 yaz_log(LOG_LOG, "from: %2X %2X %2X %2X",
417 from[0], from[1], from[2], from[3]);
418 yaz_log (LOG_WARN|LOG_ERRNO, "bad unicode sequence");
426 static int scan_string(char *s_native,
427 yaz_iconv_t t_unicode, yaz_iconv_t t_utf8,
428 void (*fun)(const char *c, void *data, int num),
429 void *data, int *num)
434 ucs4_t *s0, *s = arg;
435 ucs4_t c, begin, end;
440 char *outbuf = (char *) arg;
441 char *inbuf = s_native;
442 size_t outbytesleft = sizeof(arg)-4;
443 size_t inbytesleft = strlen(s_native);
445 ret = yaz_iconv(t_unicode, &inbuf, &inbytesleft,
446 &outbuf, &outbytesleft);
447 if (ret == (size_t)(-1))
449 i = (outbuf - (char*) arg)/sizeof(ucs4_t);
453 for (i = 0; s_native[i]; i++)
454 arg[i] = s_native[i] & 255; /* ISO-8859-1 conversion */
456 arg[i] = 0; /* terminate */
457 if (s[0] == 0xfeff || s[0] == 0xfeff) /* skip byte Order Mark */
465 begin = zebra_prim_w(&s);
468 logf(LOG_FATAL, "Bad range in char-map");
472 end = zebra_prim_w(&s);
475 logf(LOG_FATAL, "Bad range in char-map");
479 for (c = begin; c <= end; c++)
481 if (scan_to_utf8 (t_utf8, &c, 1, str, sizeof(str)-1))
483 (*fun)(str, data, num ? (*num)++ : 0);
486 case '[': s++; abort(); break;
490 while (*s != ')' || s[-1] == '\\')
493 if (scan_to_utf8 (t_utf8, s0, s - s0, str, sizeof(str)-1))
495 (*fun)(str, data, num ? (*num)++ : 0);
499 c = zebra_prim_w(&s);
500 if (scan_to_utf8 (t_utf8, &c, 1, str, sizeof(str)-1))
502 (*fun)(str, data, num ? (*num)++ : 0);
508 chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
512 char line[512], *argv[50];
516 int argc, num = (int) *CHR_BASE, i;
518 yaz_iconv_t t_unicode = 0;
519 yaz_iconv_t t_utf8 = 0;
520 unsigned endian = 31;
521 const char *ucs4_native = "UCS-4";
523 if (*(char*) &endian == 31) /* little endian? */
524 ucs4_native = "UCS-4LE";
526 t_utf8 = yaz_iconv_open ("UTF-8", ucs4_native);
528 logf (LOG_DEBUG, "maptab %s open", name);
529 if (!(f = yaz_fopen(tabpath, name, "r", tabroot)))
531 logf(LOG_WARN|LOG_ERRNO, "%s", name);
534 nmem = nmem_create ();
535 res = (chrmaptab) nmem_malloc(nmem, sizeof(*res));
537 res->input = (chr_t_entry *) nmem_malloc(res->nmem, sizeof(*res->input));
538 res->input->target = (unsigned char **)
539 nmem_malloc(res->nmem, sizeof(*res->input->target) * 2);
540 res->input->target[0] = (unsigned char*) CHR_UNKNOWN;
541 res->input->target[1] = 0;
542 res->input->children = (chr_t_entry **)
543 nmem_malloc(res->nmem, sizeof(res->input) * 256);
544 for (i = 0; i < 256; i++)
546 res->input->children[i] = (chr_t_entry *)
547 nmem_malloc(res->nmem, sizeof(*res->input));
548 res->input->children[i]->children = 0;
549 res->input->children[i]->target = (unsigned char **)
550 nmem_malloc (res->nmem, 2 * sizeof(unsigned char *));
551 res->input->children[i]->target[1] = 0;
554 res->input->children[i]->target[0] = (unsigned char *)
555 nmem_malloc (res->nmem, 2 * sizeof(unsigned char));
556 res->input->children[i]->target[0][0] = i;
557 res->input->children[i]->target[0][1] = 0;
560 res->input->children[i]->target[0] = (unsigned char*) CHR_UNKNOWN;
562 res->q_input = (chr_t_entry *)
563 nmem_malloc(res->nmem, sizeof(*res->q_input));
564 res->q_input->target = 0;
565 res->q_input->children = 0;
567 for (i = *CHR_BASE; i < 256; i++)
569 res->output[(int) *CHR_SPACE] = (unsigned char *) " ";
570 res->output[(int) *CHR_UNKNOWN] = (unsigned char*) "@";
571 res->base_uppercase = 0;
573 while (!errors && (argc = readconf_line(f, &lineno, line, 512, argv, 50)))
574 if (!map_only && !yaz_matchstr(argv[0], "lowercase"))
578 logf(LOG_FATAL, "Syntax error in charmap");
581 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
584 logf(LOG_FATAL, "Bad value-set specification");
587 res->base_uppercase = num;
588 res->output[(int) *CHR_SPACE + num] = (unsigned char *) " ";
589 res->output[(int) *CHR_UNKNOWN + num] = (unsigned char*) "@";
590 num = (int) *CHR_BASE;
592 else if (!map_only && !yaz_matchstr(argv[0], "uppercase"))
594 if (!res->base_uppercase)
596 logf(LOG_FATAL, "Uppercase directive with no lowercase set");
601 logf(LOG_FATAL, "Missing arg for uppercase directive");
604 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
607 logf(LOG_FATAL, "Bad value-set specification");
611 else if (!map_only && !yaz_matchstr(argv[0], "space"))
615 logf(LOG_FATAL, "Syntax error in charmap");
618 if (scan_string(argv[1], t_unicode, t_utf8,
619 fun_addspace, res, 0) < 0)
621 logf(LOG_FATAL, "Bad space specification");
625 else if (!yaz_matchstr(argv[0], "map"))
631 logf(LOG_FATAL, "charmap directive map requires 2 args");
635 buf.string[0] = '\0';
636 if (scan_string(argv[2], t_unicode, t_utf8,
637 fun_mkstring, &buf, 0) < 0)
639 logf(LOG_FATAL, "Bad map target");
642 if (scan_string(argv[1], t_unicode, t_utf8,
643 fun_add_map, &buf, 0) < 0)
645 logf(LOG_FATAL, "Bad map source");
649 else if (!yaz_matchstr(argv[0], "qmap"))
655 logf(LOG_FATAL, "charmap directive qmap requires 2 args");
659 buf.string[0] = '\0';
660 if (scan_string(argv[2], t_unicode, t_utf8,
661 fun_mkstring, &buf, 0) < 0)
663 logf(LOG_FATAL, "Bad qmap target");
666 if (scan_string(argv[1], t_unicode, t_utf8,
667 fun_add_qmap, &buf, 0) < 0)
669 logf(LOG_FATAL, "Bad qmap source");
673 else if (!yaz_matchstr(argv[0], "encoding"))
676 * Fix me. When t_unicode==0 and use encoding directive in *.chr file the beheviour of the
677 * zebra need to comment next part of code.
683 yaz_iconv_close (t_unicode);
684 t_unicode = yaz_iconv_open (ucs4_native, argv[1]);
687 * Fix me. It is additional staff for conversion of characters from local encoding
688 * of *.chr file to UTF-8 (internal encoding).
689 * NOTE: The derective encoding must be first directive in *.chr file.
691 /* For whatever reason Oleg enabled this.. */
694 yaz_iconv_close(t_utf8);
695 t_utf8 = yaz_iconv_open ("UTF-8", argv[1]);
700 logf(LOG_WARN, "Syntax error at '%s' in %s", line, name);
706 chrmaptab_destroy(res);
709 logf (LOG_DEBUG, "maptab %s close %d errors", name, errors);
711 yaz_iconv_close(t_utf8);
713 yaz_iconv_close(t_unicode);
717 void chrmaptab_destroy(chrmaptab tab)
720 nmem_destroy (tab->nmem);