1 /* This file is part of the YAZ toolkit.
2 * Copyright (C) Index Data
3 * See the file LICENSE for details.
6 * \file zoom-memcached.c
7 * \brief Implements query/record caching using memcached
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
21 #include <yaz/diagbib1.h>
23 #if HAVE_LIBMEMCACHED_MEMCACHED_H
24 #if HAVE_MEMCACHED_RETURN_T
26 typedef memcached_return memcached_return_t;
30 void ZOOM_memcached_init(ZOOM_connection c)
32 #if HAVE_LIBMEMCACHED_MEMCACHED_H
37 void ZOOM_memcached_destroy(ZOOM_connection c)
39 #if HAVE_LIBMEMCACHED_MEMCACHED_H
41 memcached_free(c->mc_st);
45 #if HAVE_LIBMEMCACHED_MEMCACHED_H
46 /* memcached wrapper.. Because memcached function do not exist in older libs */
47 static memcached_st *yaz_memcached_wrap(const char *conf)
49 #if HAVE_MEMCACHED_FUNC
50 return memcached(conf, strlen(conf));
54 memcached_st *mc = memcached_create(0);
55 NMEM nmem = nmem_create();
56 memcached_return_t rc;
58 nmem_strsplit_blank(nmem, conf, &darray, &num);
59 for (i = 0; mc && i < num; i++)
61 if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
63 char *host = darray[i] + 9;
64 char *port = strchr(host, ':');
65 char *weight = strstr(host, "/?");
73 rc = memcached_server_add(mc, host, port ? atoi(port) : 11211);
74 yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
75 host, (unsigned) rc, memcached_strerror(mc, rc));
76 if (rc != MEMCACHED_SUCCESS)
95 int ZOOM_memcached_configure(ZOOM_connection c)
98 #if HAVE_LIBMEMCACHED_MEMCACHED_H
101 memcached_free(c->mc_st);
105 val = ZOOM_options_get(c->options, "memcached");
108 #if HAVE_LIBMEMCACHED_MEMCACHED_H
109 c->mc_st = yaz_memcached_wrap(val);
112 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
113 "could not create memcached");
116 memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
118 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
125 #if HAVE_LIBMEMCACHED_MEMCACHED_H
126 static void wrbuf_vary_puts(WRBUF w, const char *v)
132 wrbuf_sha1_puts(w, v, 1);
142 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
144 #if HAVE_LIBMEMCACHED_MEMCACHED_H
145 ZOOM_connection c = r->connection;
147 r->mc_key = wrbuf_alloc();
148 wrbuf_puts(r->mc_key, "1;");
149 wrbuf_vary_puts(r->mc_key, c->host_port);
150 wrbuf_puts(r->mc_key, ";");
151 wrbuf_vary_puts(r->mc_key, ZOOM_resultset_option_get(r, "extraArgs"));
152 wrbuf_puts(r->mc_key, ";");
153 wrbuf_vary_puts(r->mc_key, c->user);
154 wrbuf_puts(r->mc_key, ";");
155 wrbuf_vary_puts(r->mc_key, c->group);
156 wrbuf_puts(r->mc_key, ";");
158 wrbuf_sha1_puts(r->mc_key, c->password, 1);
159 wrbuf_puts(r->mc_key, ";");
161 WRBUF w = wrbuf_alloc();
162 ZOOM_query_get_hash(q, w);
163 wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
166 wrbuf_puts(r->mc_key, ";");
167 wrbuf_vary_puts(r->mc_key, r->req_facets);
171 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
173 #if HAVE_LIBMEMCACHED_MEMCACHED_H
174 if (c->mc_st && resultset->live_set == 0)
178 memcached_return_t rc;
179 char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
180 wrbuf_len(resultset->mc_key),
181 &v_len, &flags, &rc);
182 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
186 size_t lead_len = strlen(v) + 1;
188 resultset->size = odr_atoi(v);
190 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
191 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
193 if (v_len > lead_len)
195 Z_OtherInformation *oi = 0;
196 int oi_len = v_len - lead_len;
197 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
198 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
200 yaz_log(YLOG_WARN, "oi decoding failed");
204 ZOOM_handle_search_result(c, resultset, oi);
205 ZOOM_handle_facet_result(c, resultset, oi);
208 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
209 ZOOM_connection_put_event(c, event);
210 resultset->live_set = 1;
216 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
217 Z_OtherInformation *oi, const char *precision)
219 #if HAVE_LIBMEMCACHED_MEMCACHED_H
220 if (c->mc_st && resultset->live_set == 0)
223 memcached_return_t rc;
224 time_t expiration = 36000;
226 ODR odr = odr_createmem(ODR_ENCODE);
231 str = odr_malloc(odr, 20 + strlen(precision));
232 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
233 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
236 z_OtherInformation(odr, &oi, 0, 0);
237 oi_buf = odr_getbuf(odr, &oi_len, 0);
239 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
242 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
244 rc = memcached_set(c->mc_st,
245 wrbuf_buf(resultset->mc_key),
246 wrbuf_len(resultset->mc_key),
247 key, strlen(str) + 1 + oi_len, expiration, flags);
248 yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
249 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
250 memcached_strerror(c->mc_st, rc));
256 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
258 const char *syntax, const char *elementSetName,
260 Z_SRW_diagnostic *diag)
262 #if HAVE_LIBMEMCACHED_MEMCACHED_H
263 if (r->connection->mc_st &&
264 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
266 WRBUF k = wrbuf_alloc();
267 WRBUF rec_sha1 = wrbuf_alloc();
269 memcached_return_t rc;
270 time_t expiration = 36000;
271 ODR odr = odr_createmem(ODR_ENCODE);
275 z_NamePlusRecord(odr, &npr, 0, 0);
276 rec_buf = odr_getbuf(odr, &rec_len, 0);
278 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
279 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
280 syntax ? syntax : "",
281 elementSetName ? elementSetName : "",
282 schema ? schema : "");
284 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
286 rc = memcached_set(r->connection->mc_st,
287 wrbuf_buf(k), wrbuf_len(k),
288 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
291 yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
292 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
293 memcached_strerror(r->connection->mc_st, rc));
295 rc = memcached_add(r->connection->mc_st,
296 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
300 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
301 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
302 memcached_strerror(r->connection->mc_st, rc));
306 wrbuf_destroy(rec_sha1);
311 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
313 const char *elementSetName,
316 #if HAVE_LIBMEMCACHED_MEMCACHED_H
317 if (r->connection && r->connection->mc_st)
319 WRBUF k = wrbuf_alloc();
323 memcached_return_t rc;
325 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
326 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
327 syntax ? syntax : "",
328 elementSetName ? elementSetName : "",
329 schema ? schema : "");
331 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
332 sha1_buf = memcached_get(r->connection->mc_st,
333 wrbuf_buf(k), wrbuf_len(k),
334 &sha1_len, &flags, &rc);
342 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
343 v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
344 &v_len, &flags, &rc);
348 Z_NamePlusRecord *npr = 0;
350 odr_setbuf(r->odr, v_buf, v_len, 0);
351 z_NamePlusRecord(r->odr, &npr, 0, 0);
354 yaz_log(YLOG_LOG, "returned memcached copy");
366 * c-file-style: "Stroustrup"
367 * indent-tabs-mode: nil
369 * vim: shiftwidth=4 tabstop=8 expandtab