1 /* This file is part of the YAZ toolkit.
2 * Copyright (C) Index Data
3 * See the file LICENSE for details.
6 * \file zoom-memcached.c
7 * \brief Implements query/record caching using memcached
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
21 #include <yaz/diagbib1.h>
23 void ZOOM_memcached_init(ZOOM_connection c)
31 c->expire_search = 600;
32 c->expire_record = 1200;
35 void ZOOM_memcached_destroy(ZOOM_connection c)
39 memcached_free(c->mc_st);
43 redisFree(c->redis_c);
48 static memcached_st *create_memcached(const char *conf,
49 int *expire_search, int *expire_record)
53 memcached_st *mc = memcached_create(0);
54 NMEM nmem = nmem_create();
55 memcached_return_t rc;
57 nmem_strsplit_blank(nmem, conf, &darray, &num);
58 for (i = 0; mc && i < num; i++)
60 if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
62 char *host = darray[i] + 9;
63 char *port = strchr(host, ':');
64 char *weight = strstr(host, "/?");
72 rc = memcached_server_add(mc, host, port ? atoi(port) : 11211);
73 yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
74 host, (unsigned) rc, memcached_strerror(mc, rc));
75 if (rc != MEMCACHED_SUCCESS)
81 else if (!yaz_strncasecmp(darray[i], "--EXPIRE=", 9))
83 *expire_search = atoi(darray[i] + 9);
84 *expire_record = 600 + *expire_search;
99 static redisContext *create_redis(const char *conf,
100 int *expire_search, int *expire_record)
104 NMEM nmem = nmem_create();
105 redisContext *context = 0;
107 nmem_strsplit_blank(nmem, conf, &darray, &num);
108 for (i = 0; i < num; i++)
110 if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
112 struct timeval timeout = { 1, 500000 }; /* 1.5 seconds */
113 char *host = darray[i] + 9;
114 char *port = strchr(host, ':');
117 context = redisConnectWithTimeout(host,
118 port ? atoi(port) : 6379,
121 else if (!yaz_strncasecmp(darray[i], "--EXPIRE=", 9))
123 *expire_search = atoi(darray[i] + 9);
124 *expire_record = 600 + *expire_search;
132 int ZOOM_memcached_configure(ZOOM_connection c)
138 redisFree(c->redis_c);
142 #if HAVE_LIBMEMCACHED
145 memcached_free(c->mc_st);
150 val = ZOOM_options_get(c->options, "redis");
154 c->redis_c = create_redis(val,
155 &c->expire_search, &c->expire_record);
156 if (c->redis_c == 0 || c->redis_c->err)
158 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
159 "could not create redis");
162 return 0; /* don't bother with memcached if redis is enabled */
164 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
168 val = ZOOM_options_get(c->options, "memcached");
171 #if HAVE_LIBMEMCACHED
172 c->mc_st = create_memcached(val, &c->expire_search, &c->expire_record);
175 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
176 "could not create memcached");
179 memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
181 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
189 static void wrbuf_vary_puts(WRBUF w, const char *v)
195 wrbuf_sha1_puts(w, v, 1);
205 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
208 ZOOM_connection c = r->connection;
210 r->mc_key = wrbuf_alloc();
211 wrbuf_puts(r->mc_key, "1;");
212 wrbuf_vary_puts(r->mc_key, c->host_port);
213 wrbuf_puts(r->mc_key, ";");
214 wrbuf_vary_puts(r->mc_key, ZOOM_resultset_option_get(r, "extraArgs"));
215 wrbuf_puts(r->mc_key, ";");
216 wrbuf_vary_puts(r->mc_key, c->user);
217 wrbuf_puts(r->mc_key, ";");
218 wrbuf_vary_puts(r->mc_key, c->group);
219 wrbuf_puts(r->mc_key, ";");
221 wrbuf_sha1_puts(r->mc_key, c->password, 1);
222 wrbuf_puts(r->mc_key, ";");
224 WRBUF w = wrbuf_alloc();
225 ZOOM_query_get_hash(q, w);
226 wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
229 wrbuf_puts(r->mc_key, ";");
230 wrbuf_vary_puts(r->mc_key, r->req_facets);
234 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
237 if (c->redis_c && resultset->live_set == 0)
243 argv[1] = wrbuf_cstr(resultset->mc_key);
245 reply = redisCommandArgv(c->redis_c, 2, argv, 0);
246 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
247 if (reply && reply->type == REDIS_REPLY_STRING)
249 char *v = reply->str;
250 int v_len = reply->len;
252 size_t lead_len = strlen(v) + 1;
254 resultset->size = odr_atoi(v);
256 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
257 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
259 if (v_len > lead_len)
261 Z_OtherInformation *oi = 0;
262 int oi_len = v_len - lead_len;
263 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
264 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
266 yaz_log(YLOG_WARN, "oi decoding failed");
267 freeReplyObject(reply);
270 ZOOM_handle_search_result(c, resultset, oi);
271 ZOOM_handle_facet_result(c, resultset, oi);
273 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
274 ZOOM_connection_put_event(c, event);
275 resultset->live_set = 1;
278 freeReplyObject(reply);
281 #if HAVE_LIBMEMCACHED
282 if (c->mc_st && resultset->live_set == 0)
286 memcached_return_t rc;
287 char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
288 wrbuf_len(resultset->mc_key),
289 &v_len, &flags, &rc);
290 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
294 size_t lead_len = strlen(v) + 1;
296 resultset->size = odr_atoi(v);
298 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
299 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
301 if (v_len > lead_len)
303 Z_OtherInformation *oi = 0;
304 int oi_len = v_len - lead_len;
305 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
306 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
308 yaz_log(YLOG_WARN, "oi decoding failed");
312 ZOOM_handle_search_result(c, resultset, oi);
313 ZOOM_handle_facet_result(c, resultset, oi);
316 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
317 ZOOM_connection_put_event(c, event);
318 resultset->live_set = 1;
325 static void expire_redis(redisContext *redis_c,
326 const char *buf, size_t len, int exp)
333 sprintf(key_val, "%d", exp);
340 argvlen[2] = strlen(key_val);
341 reply = redisCommandArgv(redis_c, 3, argv, argvlen);
342 freeReplyObject(reply);
346 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
347 Z_OtherInformation *oi, const char *precision)
350 if (c->redis_c && resultset->live_set == 0)
353 ODR odr = odr_createmem(ODR_ENCODE);
358 str = odr_malloc(odr, 20 + strlen(precision));
359 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
360 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
363 z_OtherInformation(odr, &oi, 0, 0);
364 oi_buf = odr_getbuf(odr, &oi_len, 0);
366 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
369 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
377 argv[1] = wrbuf_buf(resultset->mc_key);
378 argvlen[1] = wrbuf_len(resultset->mc_key);
380 argvlen[2] = strlen(str) + 1 + oi_len;
381 reply = redisCommandArgv(c->redis_c, 3, argv, argvlen);
382 freeReplyObject(reply);
384 expire_redis(c->redis_c,
385 wrbuf_buf(resultset->mc_key),
386 wrbuf_len(resultset->mc_key),
391 #if HAVE_LIBMEMCACHED
392 if (c->mc_st && resultset->live_set == 0)
395 memcached_return_t rc;
397 ODR odr = odr_createmem(ODR_ENCODE);
402 str = odr_malloc(odr, 20 + strlen(precision));
403 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
404 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
407 z_OtherInformation(odr, &oi, 0, 0);
408 oi_buf = odr_getbuf(odr, &oi_len, 0);
410 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
413 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
415 rc = memcached_set(c->mc_st,
416 wrbuf_buf(resultset->mc_key),
417 wrbuf_len(resultset->mc_key),
418 key, strlen(str) + 1 + oi_len,
419 c->expire_search, flags);
420 yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
421 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
422 memcached_strerror(c->mc_st, rc));
428 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
430 const char *syntax, const char *elementSetName,
432 Z_SRW_diagnostic *diag)
435 if (r->connection->redis_c &&
436 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
438 WRBUF k = wrbuf_alloc();
439 WRBUF rec_sha1 = wrbuf_alloc();
440 ODR odr = odr_createmem(ODR_ENCODE);
447 z_NamePlusRecord(odr, &npr, 0, 0);
448 rec_buf = odr_getbuf(odr, &rec_len, 0);
450 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
451 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
452 syntax ? syntax : "",
453 elementSetName ? elementSetName : "",
454 schema ? schema : "");
456 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
460 argv[1] = wrbuf_buf(k);
461 argvlen[1] = wrbuf_len(k);
462 argv[2] = wrbuf_buf(rec_sha1);
463 argvlen[2] = wrbuf_len(rec_sha1);
465 reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
466 yaz_log(YLOG_LOG, "Store record key=%s val=%s",
467 wrbuf_cstr(k), wrbuf_cstr(rec_sha1));
468 freeReplyObject(reply);
470 expire_redis(r->connection->redis_c, argv[1], argvlen[1],
471 r->connection->expire_search);
473 argv[1] = wrbuf_buf(rec_sha1);
474 argvlen[1] = wrbuf_len(rec_sha1);
476 argvlen[2] = rec_len;
478 reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
479 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d",
480 wrbuf_cstr(rec_sha1), rec_len);
481 freeReplyObject(reply);
483 expire_redis(r->connection->redis_c, argv[1], argvlen[1],
484 r->connection->expire_record);
488 wrbuf_destroy(rec_sha1);
491 #if HAVE_LIBMEMCACHED
492 if (r->connection->mc_st &&
493 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
495 WRBUF k = wrbuf_alloc();
496 WRBUF rec_sha1 = wrbuf_alloc();
498 memcached_return_t rc;
499 ODR odr = odr_createmem(ODR_ENCODE);
503 z_NamePlusRecord(odr, &npr, 0, 0);
504 rec_buf = odr_getbuf(odr, &rec_len, 0);
506 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
507 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
508 syntax ? syntax : "",
509 elementSetName ? elementSetName : "",
510 schema ? schema : "");
512 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
514 rc = memcached_set(r->connection->mc_st,
515 wrbuf_buf(k), wrbuf_len(k),
516 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
517 r->connection->expire_search, flags);
519 yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
520 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
521 memcached_strerror(r->connection->mc_st, rc));
523 rc = memcached_add(r->connection->mc_st,
524 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
526 r->connection->expire_record, flags);
528 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
529 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
530 memcached_strerror(r->connection->mc_st, rc));
534 wrbuf_destroy(rec_sha1);
539 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
541 const char *elementSetName,
545 if (r->connection && r->connection->redis_c)
547 WRBUF k = wrbuf_alloc();
552 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
553 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
554 syntax ? syntax : "",
555 elementSetName ? elementSetName : "",
556 schema ? schema : "");
558 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
561 argv[1] = wrbuf_buf(k);
562 argvlen[1] = wrbuf_len(k);
563 reply1 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
566 if (reply1 && reply1->type == REDIS_REPLY_STRING)
569 char *sha1_buf = reply1->str;
570 int sha1_len = reply1->len;
572 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
577 argvlen[1] = sha1_len;
579 reply2 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
580 if (reply2 && reply2->type == REDIS_REPLY_STRING)
582 Z_NamePlusRecord *npr = 0;
583 char *v_buf = reply2->str;
584 int v_len = reply2->len;
586 odr_setbuf(r->odr, v_buf, v_len, 0);
587 z_NamePlusRecord(r->odr, &npr, 0, 0);
589 yaz_log(YLOG_LOG, "returned redis copy");
590 freeReplyObject(reply2);
591 freeReplyObject(reply1);
594 freeReplyObject(reply2);
596 freeReplyObject(reply1);
599 #if HAVE_LIBMEMCACHED
600 if (r->connection && r->connection->mc_st)
602 WRBUF k = wrbuf_alloc();
606 memcached_return_t rc;
608 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
609 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
610 syntax ? syntax : "",
611 elementSetName ? elementSetName : "",
612 schema ? schema : "");
614 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
615 sha1_buf = memcached_get(r->connection->mc_st,
616 wrbuf_buf(k), wrbuf_len(k),
617 &sha1_len, &flags, &rc);
625 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
626 v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
627 &v_len, &flags, &rc);
631 Z_NamePlusRecord *npr = 0;
633 odr_setbuf(r->odr, v_buf, v_len, 0);
634 z_NamePlusRecord(r->odr, &npr, 0, 0);
637 yaz_log(YLOG_LOG, "returned memcached copy");
649 * c-file-style: "Stroustrup"
650 * indent-tabs-mode: nil
652 * vim: shiftwidth=4 tabstop=8 expandtab