1 /* This file is part of the YAZ toolkit.
2 * Copyright (C) 1995-2013 Index Data
3 * See the file LICENSE for details.
7 * \brief Implements CQL transform (CQL to RPN conversion).
9 * Evaluation order of rules:
26 #include <yaz/rpn2cql.h>
27 #include <yaz/xmalloc.h>
28 #include <yaz/diagsrw.h>
29 #include <yaz/tokenizer.h>
30 #include <yaz/wrbuf.h>
31 #include <yaz/z-core.h>
32 #include <yaz/matchstr.h>
33 #include <yaz/oid_db.h>
36 struct cql_prop_entry {
39 Z_AttributeList attr_list;
40 struct cql_prop_entry *next;
43 struct cql_transform_t_ {
44 struct cql_prop_entry *entry;
45 yaz_tok_cfg_t tok_cfg;
53 cql_transform_t cql_transform_create(void)
55 cql_transform_t ct = (cql_transform_t) xmalloc(sizeof(*ct));
56 ct->tok_cfg = yaz_tok_cfg_create();
57 ct->w = wrbuf_alloc();
61 ct->nmem = nmem_create();
65 static int cql_transform_parse_tok_line(cql_transform_t ct,
70 Z_AttributeElement *ae[20];
71 int ret = 0; /* 0=OK, != 0 FAIL */
75 while (t == YAZ_TOK_STRING && ae_num < 20)
77 WRBUF type_str = wrbuf_alloc();
79 Z_AttributeElement *elem = 0;
80 const char *value_str = 0;
81 /* attset type=value OR type=value */
83 elem = (Z_AttributeElement *) nmem_malloc(ct->nmem, sizeof(*elem));
84 elem->attributeSet = 0;
86 wrbuf_puts(ct->w, yaz_tok_parse_string(tp));
87 wrbuf_puts(type_str, yaz_tok_parse_string(tp));
91 wrbuf_destroy(type_str);
93 wrbuf_destroy(set_str);
96 if (t == YAZ_TOK_STRING)
98 wrbuf_puts(ct->w, " ");
99 wrbuf_puts(ct->w, yaz_tok_parse_string(tp));
103 yaz_string_to_oid_nmem(yaz_oid_std(), CLASS_ATTSET,
104 wrbuf_cstr(set_str), ct->nmem);
106 type_str = wrbuf_alloc();
107 wrbuf_puts(type_str, yaz_tok_parse_string(tp));
108 t = yaz_tok_move(tp);
110 elem->attributeType = nmem_intdup(ct->nmem, 0);
111 if (sscanf(wrbuf_cstr(type_str), ODR_INT_PRINTF, elem->attributeType)
114 wrbuf_destroy(type_str);
116 wrbuf_destroy(set_str);
117 yaz_log(YLOG_WARN, "Expected numeric attribute type");
122 wrbuf_destroy(type_str);
124 wrbuf_destroy(set_str);
128 yaz_log(YLOG_WARN, "Expected = after after attribute type");
132 t = yaz_tok_move(tp);
133 if (t != YAZ_TOK_STRING) /* value */
135 yaz_log(YLOG_WARN, "Missing attribute value");
139 value_str = yaz_tok_parse_string(tp);
140 if (yaz_isdigit(*value_str))
142 elem->which = Z_AttributeValue_numeric;
143 elem->value.numeric =
144 nmem_intdup(ct->nmem, atoi(value_str));
148 Z_ComplexAttribute *ca = (Z_ComplexAttribute *)
149 nmem_malloc(ct->nmem, sizeof(*ca));
150 elem->which = Z_AttributeValue_complex;
151 elem->value.complex = ca;
153 ca->list = (Z_StringOrNumeric **)
154 nmem_malloc(ct->nmem, sizeof(Z_StringOrNumeric *));
155 ca->list[0] = (Z_StringOrNumeric *)
156 nmem_malloc(ct->nmem, sizeof(Z_StringOrNumeric));
157 ca->list[0]->which = Z_StringOrNumeric_string;
158 ca->list[0]->u.string = nmem_strdup(ct->nmem, value_str);
159 ca->num_semanticAction = 0;
160 ca->semanticAction = 0;
162 wrbuf_puts(ct->w, "=");
163 wrbuf_puts(ct->w, yaz_tok_parse_string(tp));
164 t = yaz_tok_move(tp);
165 wrbuf_puts(ct->w, " ");
168 if (ret == 0) /* OK? */
170 struct cql_prop_entry **pp = &ct->entry;
173 *pp = (struct cql_prop_entry *) xmalloc(sizeof(**pp));
174 (*pp)->pattern = xstrdup(pattern);
175 (*pp)->value = xstrdup(wrbuf_cstr(ct->w));
177 (*pp)->attr_list.num_attributes = ae_num;
179 (*pp)->attr_list.attributes = 0;
182 (*pp)->attr_list.attributes = (Z_AttributeElement **)
183 nmem_malloc(ct->nmem,
184 ae_num * sizeof(Z_AttributeElement *));
185 memcpy((*pp)->attr_list.attributes, ae,
186 ae_num * sizeof(Z_AttributeElement *));
192 ODR pr = odr_createmem(ODR_PRINT);
193 Z_AttributeList *alp = &(*pp)->attr_list;
194 odr_setprint(pr, yaz_log_file());
195 z_AttributeList(pr, &alp, 0, 0);
203 int cql_transform_define_pattern(cql_transform_t ct, const char *pattern,
207 yaz_tok_parse_t tp = yaz_tok_parse_buf(ct->tok_cfg, value);
208 yaz_tok_cfg_single_tokens(ct->tok_cfg, "=");
209 r = cql_transform_parse_tok_line(ct, pattern, tp);
210 yaz_tok_parse_destroy(tp);
214 cql_transform_t cql_transform_open_FILE(FILE *f)
216 cql_transform_t ct = cql_transform_create();
219 yaz_tok_cfg_single_tokens(ct->tok_cfg, "=");
221 while (fgets(line, sizeof(line)-1, f))
223 yaz_tok_parse_t tp = yaz_tok_parse_buf(ct->tok_cfg, line);
226 t = yaz_tok_move(tp);
227 if (t == YAZ_TOK_STRING)
229 char * pattern = xstrdup(yaz_tok_parse_string(tp));
230 t = yaz_tok_move(tp);
233 yaz_tok_parse_destroy(tp);
234 cql_transform_close(ct);
237 if (cql_transform_parse_tok_line(ct, pattern, tp))
239 yaz_tok_parse_destroy(tp);
240 cql_transform_close(ct);
245 else if (t != YAZ_TOK_EOF)
247 yaz_tok_parse_destroy(tp);
248 cql_transform_close(ct);
251 yaz_tok_parse_destroy(tp);
256 void cql_transform_close(cql_transform_t ct)
258 struct cql_prop_entry *pe;
264 struct cql_prop_entry *pe_next = pe->next;
271 yaz_tok_cfg_destroy(ct->tok_cfg);
272 wrbuf_destroy(ct->w);
273 nmem_destroy(ct->nmem);
277 cql_transform_t cql_transform_open_fname(const char *fname)
280 FILE *f = fopen(fname, "r");
283 ct = cql_transform_open_FILE(f);
289 struct Z_AttributeElement {
290 Z_AttributeSetId *attributeSet; /* OPT */
295 Z_ComplexAttribute *complex;
296 #define Z_AttributeValue_numeric 1
297 #define Z_AttributeValue_complex 2
302 static int compare_attr(Z_AttributeElement *a, Z_AttributeElement *b)
304 ODR odr_a = odr_createmem(ODR_ENCODE);
305 ODR odr_b = odr_createmem(ODR_ENCODE);
310 z_AttributeElement(odr_a, &a, 0, 0);
311 z_AttributeElement(odr_b, &b, 0, 0);
313 buf_a = odr_getbuf(odr_a, &len_a, 0);
314 buf_b = odr_getbuf(odr_b, &len_b, 0);
316 ret = yaz_memcmp(buf_a, buf_b, len_a, len_b);
323 const char *cql_lookup_reverse(cql_transform_t ct,
324 const char *category,
325 Z_AttributeList *attributes)
327 struct cql_prop_entry *e;
328 size_t clen = strlen(category);
329 for (e = ct->entry; e; e = e->next)
331 if (!strncmp(e->pattern, category, clen))
333 /* category matches.. See if attributes in pattern value
334 are all listed in actual attributes */
336 for (i = 0; i < e->attr_list.num_attributes; i++)
338 /* entry attribute */
339 Z_AttributeElement *e_ae = e->attr_list.attributes[i];
341 for (j = 0; j < attributes->num_attributes; j++)
343 /* actual attribute */
344 Z_AttributeElement *a_ae = attributes->attributes[j];
345 int r = compare_attr(e_ae, a_ae);
349 if (j == attributes->num_attributes)
350 break; /* i was not found at all.. try next pattern */
353 if (i == e->attr_list.num_attributes)
354 return e->pattern + clen;
360 static const char *cql_lookup_property(cql_transform_t ct,
361 const char *pat1, const char *pat2,
365 struct cql_prop_entry *e;
367 if (pat1 && pat2 && pat3)
368 sprintf(pattern, "%.39s.%.39s.%.39s", pat1, pat2, pat3);
369 else if (pat1 && pat2)
370 sprintf(pattern, "%.39s.%.39s", pat1, pat2);
371 else if (pat1 && pat3)
372 sprintf(pattern, "%.39s.%.39s", pat1, pat3);
374 sprintf(pattern, "%.39s", pat1);
378 for (e = ct->entry; e; e = e->next)
380 if (!cql_strcmp(e->pattern, pattern))
386 int cql_pr_attr_uri(cql_transform_t ct, const char *category,
387 const char *uri, const char *val, const char *default_val,
388 void (*pr)(const char *buf, void *client_data),
393 const char *eval = val ? val : default_val;
394 const char *prefix = 0;
398 struct cql_prop_entry *e;
400 for (e = ct->entry; e; e = e->next)
401 if (!memcmp(e->pattern, "set.", 4) && e->value &&
402 !strcmp(e->value, uri))
404 prefix = e->pattern+4;
407 /* must have a prefix now - if not it's an error */
413 res = cql_lookup_property(ct, category, prefix, eval);
414 /* we have some aliases for some relations unfortunately.. */
415 if (!res && !prefix && !strcmp(category, "relation"))
417 if (!strcmp(val, "=="))
418 res = cql_lookup_property(ct, category, prefix, "exact");
419 if (!strcmp(val, "="))
420 res = cql_lookup_property(ct, category, prefix, "eq");
421 if (!strcmp(val, "<="))
422 res = cql_lookup_property(ct, category, prefix, "le");
423 if (!strcmp(val, ">="))
424 res = cql_lookup_property(ct, category, prefix, "ge");
427 res = cql_lookup_property(ct, category, prefix, "*");
433 const char *cp0 = res, *cp1;
434 while ((cp1 = strchr(cp0, '=')))
437 while (*cp1 && *cp1 != ' ')
439 if (cp1 - cp0 >= (ptrdiff_t) sizeof(buf))
441 memcpy(buf, cp0, cp1 - cp0);
443 (*pr)("@attr ", client_data);
445 for (i = 0; buf[i]; i++)
448 (*pr)(eval, client_data);
454 (*pr)(tmp, client_data);
457 (*pr)(" ", client_data);
465 if (errcode && !ct->error)
469 ct->addinfo = xstrdup(val);
476 int cql_pr_attr(cql_transform_t ct, const char *category,
477 const char *val, const char *default_val,
478 void (*pr)(const char *buf, void *client_data),
482 return cql_pr_attr_uri(ct, category, 0 /* uri */,
483 val, default_val, pr, client_data, errcode);
487 static void cql_pr_int(int val,
488 void (*pr)(const char *buf, void *client_data),
491 char buf[21]; /* enough characters to 2^64 */
492 sprintf(buf, "%d", val);
493 (*pr)(buf, client_data);
494 (*pr)(" ", client_data);
498 static int cql_pr_prox(cql_transform_t ct, struct cql_node *mods,
499 void (*pr)(const char *buf, void *client_data),
505 int proxrel = 2; /* less than or equal */
506 int unit = 2; /* word */
510 const char *name = mods->u.st.index;
511 const char *term = mods->u.st.term;
512 const char *relation = mods->u.st.relation;
514 if (!strcmp(name, "distance")) {
515 distance = strtol(term, (char**) 0, 0);
516 if (!strcmp(relation, "="))
518 else if (!strcmp(relation, ">"))
520 else if (!strcmp(relation, "<"))
522 else if (!strcmp(relation, ">="))
524 else if (!strcmp(relation, "<="))
526 else if (!strcmp(relation, "<>"))
530 ct->error = YAZ_SRW_UNSUPP_PROX_RELATION;
531 ct->addinfo = xstrdup(relation);
535 else if (!strcmp(name, "ordered"))
537 else if (!strcmp(name, "unordered"))
539 else if (!strcmp(name, "unit"))
541 if (!strcmp(term, "word"))
543 else if (!strcmp(term, "sentence"))
545 else if (!strcmp(term, "paragraph"))
547 else if (!strcmp(term, "element"))
551 ct->error = YAZ_SRW_UNSUPP_PROX_UNIT;
552 ct->addinfo = xstrdup(term);
558 ct->error = YAZ_SRW_UNSUPP_BOOLEAN_MODIFIER;
559 ct->addinfo = xstrdup(name);
562 mods = mods->u.st.modifiers;
566 distance = (unit == 2) ? 1 : 0;
568 cql_pr_int(exclusion, pr, client_data);
569 cql_pr_int(distance, pr, client_data);
570 cql_pr_int(ordered, pr, client_data);
571 cql_pr_int(proxrel, pr, client_data);
572 (*pr)("k ", client_data);
573 cql_pr_int(unit, pr, client_data);
578 /* ### checks for CQL relation-name rather than Type-1 attribute */
579 static int has_modifier(struct cql_node *cn, const char *name) {
580 struct cql_node *mod;
581 for (mod = cn->u.st.modifiers; mod != 0; mod = mod->u.st.modifiers) {
582 if (!strcmp(mod->u.st.index, name))
589 static void emit_term(cql_transform_t ct,
591 const char *term, int length,
592 void (*pr)(const char *buf, void *client_data),
596 const char *ns = cn->u.st.index_uri;
598 int process_term = 1;
600 if (has_modifier(cn, "regexp"))
602 else if (has_modifier(cn, "unmasked"))
604 else if (cql_lookup_property(ct, "truncation", 0, "cql"))
607 cql_pr_attr(ct, "truncation", "cql", 0,
608 pr, client_data, YAZ_SRW_MASKING_CHAR_UNSUPP);
610 assert(cn->which == CQL_NODE_ST);
613 { /* convert term via truncation.things */
616 for (i = 0; i < length; i++)
618 if (term[i] == '\\' && i < length - 1)
627 else if (i == length - 1)
633 else if (i == length - 1)
646 cql_pr_attr(ct, "position", "firstAndLast", 0,
647 pr, client_data, YAZ_SRW_ANCHORING_CHAR_IN_UNSUPP_POSITION);
651 else if (anchor == 1)
653 cql_pr_attr(ct, "position", "first", 0,
654 pr, client_data, YAZ_SRW_ANCHORING_CHAR_IN_UNSUPP_POSITION);
658 else if (anchor == 2)
660 cql_pr_attr(ct, "position", "last", 0,
661 pr, client_data, YAZ_SRW_ANCHORING_CHAR_IN_UNSUPP_POSITION);
666 cql_pr_attr(ct, "position", "any", 0,
667 pr, client_data, YAZ_SRW_ANCHORING_CHAR_IN_UNSUPP_POSITION);
671 if (trunc == 3 && cql_pr_attr(ct, "truncation",
672 "both", 0, pr, client_data, 0))
677 else if (trunc == 1 && cql_pr_attr(ct, "truncation",
678 "left", 0, pr, client_data, 0))
683 else if (trunc == 2 && cql_pr_attr(ct, "truncation", "right", 0,
691 cql_pr_attr(ct, "truncation", "none", 0,
695 cql_pr_attr(ct, "truncation", "z3958", 0,
696 pr, client_data, YAZ_SRW_MASKING_CHAR_UNSUPP);
699 cql_pr_attr_uri(ct, "index", ns,
700 cn->u.st.index, "serverChoice",
701 pr, client_data, YAZ_SRW_UNSUPP_INDEX);
703 if (cn->u.st.modifiers)
705 struct cql_node *mod = cn->u.st.modifiers;
706 for (; mod; mod = mod->u.st.modifiers)
708 cql_pr_attr(ct, "relationModifier", mod->u.st.index, 0,
709 pr, client_data, YAZ_SRW_UNSUPP_RELATION_MODIFIER);
712 (*pr)("\"", client_data);
714 for (i = 0; i < length; i++)
716 char x[2]; /* temp buffer */
717 if (term[i] == '\\' && i < length - 1)
720 if (strchr("\"\\", term[i]))
721 pr("\\", client_data);
722 if (z3958_mode && strchr("#?", term[i]))
723 pr("\\\\", client_data); /* double \\ to survive PQF parse */
728 else if (z3958_mode && term[i] == '*')
730 pr("?", client_data);
731 if (i < length - 1 && yaz_isdigit(term[i+1]))
732 pr("\\\\", client_data); /* dbl \\ to survive PQF parse */
734 else if (z3958_mode && term[i] == '?')
736 pr("#", client_data);
741 pr("\\", client_data);
742 if (z3958_mode && strchr("#?", term[i]))
743 pr("\\\\", client_data); /* dbl \\ to survive PQF parse */
751 for (i = 0; i < length; i++)
759 (*pr)("\" ", client_data);
762 static void emit_terms(cql_transform_t ct,
764 void (*pr)(const char *buf, void *client_data),
767 emit_term(ct, cn, cn->u.st.term, strlen(cn->u.st.term),
771 static void emit_wordlist(cql_transform_t ct,
773 void (*pr)(const char *buf, void *client_data),
777 const char *cp0 = cn->u.st.term;
779 const char *last_term = 0;
785 cp1 = strchr(cp0, ' ');
788 (*pr)("@", client_data);
789 (*pr)(op, client_data);
790 (*pr)(" ", client_data);
791 emit_term(ct, cn, last_term, last_length, pr, client_data);
795 last_length = cp1 - cp0;
797 last_length = strlen(cp0);
801 emit_term(ct, cn, last_term, last_length, pr, client_data);
804 void cql_transform_r(cql_transform_t ct,
806 void (*pr)(const char *buf, void *client_data),
810 struct cql_node *mods;
817 ns = cn->u.st.index_uri;
820 if (!strcmp(ns, cql_uri())
821 && cn->u.st.index && !cql_strcmp(cn->u.st.index, "resultSet"))
823 (*pr)("@set \"", client_data);
824 (*pr)(cn->u.st.term, client_data);
825 (*pr)("\" ", client_data);
833 ct->error = YAZ_SRW_UNSUPP_CONTEXT_SET;
837 cql_pr_attr(ct, "always", 0, 0, pr, client_data, 0);
838 cql_pr_attr(ct, "relation", cn->u.st.relation, 0, pr, client_data,
839 YAZ_SRW_UNSUPP_RELATION);
840 cql_pr_attr(ct, "structure", cn->u.st.relation, 0,
841 pr, client_data, YAZ_SRW_UNSUPP_COMBI_OF_RELATION_AND_TERM);
842 if (cn->u.st.relation && !cql_strcmp(cn->u.st.relation, "all"))
843 emit_wordlist(ct, cn, pr, client_data, "and");
844 else if (cn->u.st.relation && !cql_strcmp(cn->u.st.relation, "any"))
845 emit_wordlist(ct, cn, pr, client_data, "or");
847 emit_terms(ct, cn, pr, client_data);
850 (*pr)("@", client_data);
851 (*pr)(cn->u.boolean.value, client_data);
852 (*pr)(" ", client_data);
853 mods = cn->u.boolean.modifiers;
854 if (!strcmp(cn->u.boolean.value, "prox"))
856 if (!cql_pr_prox(ct, mods, pr, client_data))
861 /* Boolean modifiers other than on proximity not supported */
862 ct->error = YAZ_SRW_UNSUPP_BOOLEAN_MODIFIER;
863 ct->addinfo = xstrdup(mods->u.st.index);
867 cql_transform_r(ct, cn->u.boolean.left, pr, client_data);
868 cql_transform_r(ct, cn->u.boolean.right, pr, client_data);
871 cql_transform_r(ct, cn->u.sort.search, pr, client_data);
874 fprintf(stderr, "Fatal: impossible CQL node-type %d\n", cn->which);
879 int cql_transform(cql_transform_t ct, struct cql_node *cn,
880 void (*pr)(const char *buf, void *client_data),
883 struct cql_prop_entry *e;
884 NMEM nmem = nmem_create();
890 for (e = ct->entry; e ; e = e->next)
892 if (!cql_strncmp(e->pattern, "set.", 4))
893 cql_apply_prefix(nmem, cn, e->pattern+4, e->value);
894 else if (!cql_strcmp(e->pattern, "set"))
895 cql_apply_prefix(nmem, cn, 0, e->value);
897 cql_transform_r(ct, cn, pr, client_data);
903 int cql_transform_FILE(cql_transform_t ct, struct cql_node *cn, FILE *f)
905 return cql_transform(ct, cn, cql_fputs, f);
908 int cql_transform_buf(cql_transform_t ct, struct cql_node *cn,
911 struct cql_buf_write_info info;
917 r = cql_transform(ct, cn, cql_buf_write_handler, &info);
919 /* Attempt to write past end of buffer. For some reason, this
920 SRW diagnostic is deprecated, but it's so perfect for our
921 purposes that it would be stupid not to use it. */
923 ct->error = YAZ_SRW_TOO_MANY_CHARS_IN_QUERY;
924 sprintf(numbuf, "%ld", (long) info.max);
925 ct->addinfo = xstrdup(numbuf);
929 info.buf[info.off] = '\0';
933 int cql_transform_error(cql_transform_t ct, const char **addinfo)
935 *addinfo = ct->addinfo;
939 void cql_transform_set_error(cql_transform_t ct, int error, const char *addinfo)
942 ct->addinfo = addinfo ? xstrdup(addinfo) : 0;
949 * c-file-style: "Stroustrup"
950 * indent-tabs-mode: nil
952 * vim: shiftwidth=4 tabstop=8 expandtab