1 /* $Id: http.c,v 1.35 2007-06-26 13:01:07 adam Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
23 #include <sys/socket.h>
24 #include <sys/types.h>
40 #include <netinet/in.h>
41 #include <arpa/inet.h>
44 #include <yaz/yaz-util.h>
45 #include <yaz/comstack.h>
53 #include "http_command.h"
55 static void proxy_io(IOCHAN i, int event);
56 static struct http_channel *http_create(const char *addr);
57 static void http_destroy(IOCHAN i);
59 // If this is set, we proxy normal HTTP requests
60 static struct sockaddr_in *proxy_addr = 0;
61 static char proxy_url[256] = "";
62 static char myurl[256] = "";
63 static struct http_buf *http_buf_freelist = 0;
64 static struct http_channel *http_channel_freelist = 0;
66 struct http_channel_observer_s {
68 void (*destroy)(void *data, struct http_channel *chan);
69 struct http_channel_observer_s *next;
70 struct http_channel *chan;
73 static struct http_buf *http_buf_create()
77 if (http_buf_freelist)
79 r = http_buf_freelist;
80 http_buf_freelist = http_buf_freelist->next;
83 r = xmalloc(sizeof(struct http_buf));
90 static void http_buf_destroy(struct http_buf *b)
92 b->next = http_buf_freelist;
93 http_buf_freelist = b;
96 static void http_buf_destroy_queue(struct http_buf *b)
108 // Calculate length of chain
109 static int http_buf_len(struct http_buf *b)
112 for (; b; b = b->next)
118 static struct http_buf *http_buf_bybuf(char *b, int len)
120 struct http_buf *res = 0;
121 struct http_buf **p = &res;
126 if (tocopy > HTTP_BUF_SIZE)
127 tocopy = HTTP_BUF_SIZE;
128 *p = http_buf_create();
129 memcpy((*p)->buf, b, tocopy);
138 // Add a (chain of) buffers to the end of an existing queue.
139 static void http_buf_enqueue(struct http_buf **queue, struct http_buf *b)
142 queue = &(*queue)->next;
146 static struct http_buf *http_buf_bywrbuf(WRBUF wrbuf)
148 // Heavens to Betsy (buf)!
149 return http_buf_bybuf(wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
152 // Non-destructively collapse chain of buffers into a string (max *len)
154 static int http_buf_peek(struct http_buf *b, char *buf, int len)
157 while (b && rd < len)
159 int toread = len - rd;
162 memcpy(buf + rd, b->buf + b->offset, toread);
170 // Ddestructively munch up to len from head of queue.
171 static int http_buf_read(struct http_buf **b, char *buf, int len)
174 while ((*b) && rd < len)
176 int toread = len - rd;
177 if (toread > (*b)->len)
179 memcpy(buf + rd, (*b)->buf + (*b)->offset, toread);
181 if (toread < (*b)->len)
184 (*b)->offset += toread;
189 struct http_buf *n = (*b)->next;
190 http_buf_destroy(*b);
198 // Buffers may overlap.
199 static void urldecode(char *i, char *o)
211 sscanf(i, "%2hhx", o);
221 // Warning: Buffers may not overlap
222 void urlencode(const char *i, char *o)
226 if (strchr(" /:", *i))
228 sprintf(o, "%%%.2X", (int) *i);
238 void http_addheader(struct http_response *r, const char *name, const char *value)
240 struct http_channel *c = r->channel;
241 struct http_header *h = nmem_malloc(c->nmem, sizeof *h);
242 h->name = nmem_strdup(c->nmem, name);
243 h->value = nmem_strdup(c->nmem, value);
244 h->next = r->headers;
248 char *http_argbyname(struct http_request *r, char *name)
250 struct http_argument *p;
253 for (p = r->arguments; p; p = p->next)
254 if (!strcmp(p->name, name))
259 char *http_headerbyname(struct http_header *h, char *name)
261 for (; h; h = h->next)
262 if (!strcmp(h->name, name))
267 struct http_response *http_create_response(struct http_channel *c)
269 struct http_response *r = nmem_malloc(c->nmem, sizeof(*r));
270 strcpy(r->code, "200");
278 // Check if buf contains a package (minus payload)
279 static int package_check(const char *buf)
282 while (*buf) // Check if we have a sequence of lines terminated by an empty line
284 char *b = strstr(buf, "\r\n");
289 len += (b - buf) + 2;
297 // Check if we have a request. Return 0 or length
298 // (including trailing CRNL) FIXME: Does not deal gracefully with requests
299 // carrying payload but this is kind of OK since we will reject anything
300 // other than an empty GET
301 static int request_check(struct http_buf *queue)
305 http_buf_peek(queue, tmp, 4096);
306 return package_check(tmp);
309 struct http_response *http_parse_response_buf(struct http_channel *c, const char *buf, int len)
312 struct http_response *r = http_create_response(c);
314 struct http_header **hp = &r->headers;
318 memcpy(tmp, buf, len);
319 for (p = tmp; *p && *p != ' '; p++) // Skip HTTP version
323 for (p2 = p; *p2 && *p2 != ' ' && p2 - p < 3; p2++)
324 r->code[p2 - p] = *p2;
325 if (!(p = strstr(tmp, "\r\n")))
330 if (!(p2 = strstr(p, "\r\n")))
332 if (p == p2) // End of headers
336 struct http_header *h = *hp = nmem_malloc(c->nmem, sizeof(*h));
337 char *value = strchr(p, ':');
341 h->name = nmem_strdup(c->nmem, p);
342 while (isspace(*value))
344 if (value >= p2) // Empty header;
351 h->value = nmem_strdup(c->nmem, value);
360 struct http_request *http_parse_request(struct http_channel *c,
361 struct http_buf **queue,
364 struct http_request *r = nmem_malloc(c->nmem, sizeof(*r));
371 if (http_buf_read(queue, buf, len) < len)
379 for (p = buf, p2 = r->method; *p && *p != ' ' && p - buf < 19; p++)
383 yaz_log(YLOG_WARN, "Unexpected HTTP method in request");
388 if (!(buf = strchr(buf, ' ')))
390 yaz_log(YLOG_WARN, "Syntax error in request (1)");
394 if (!(p = strchr(buf, ' ')))
396 yaz_log(YLOG_WARN, "Syntax error in request (2)");
400 if ((p2 = strchr(buf, '?'))) // Do we have arguments?
402 r->path = nmem_strdup(c->nmem, buf);
405 r->search = nmem_strdup(c->nmem, p2);
409 struct http_argument *a;
410 char *equal = strchr(p2, '=');
411 char *eoa = strchr(p2, '&');
414 yaz_log(YLOG_WARN, "Expected '=' in argument");
418 eoa = equal + strlen(equal); // last argument
421 a = nmem_malloc(c->nmem, sizeof(struct http_argument));
423 a->name = nmem_strdup(c->nmem, p2);
424 urldecode(a->name, a->name);
425 urldecode(equal, equal);
426 a->value = nmem_strdup(c->nmem, equal);
427 a->next = r->arguments;
434 if (strncmp(buf, "HTTP/", 5))
435 strcpy(r->http_version, "1.0");
439 if (!(p = strstr(buf, "\r\n")))
443 strcpy(r->http_version, buf);
446 strcpy(c->version, r->http_version);
451 if (!(p = strstr(buf, "\r\n")))
457 struct http_header *h = nmem_malloc(c->nmem, sizeof(*h));
458 if (!(p2 = strchr(buf, ':')))
461 h->name = nmem_strdup(c->nmem, buf);
464 if (p2 >= p) // Empty header?
470 h->value = nmem_strdup(c->nmem, p2);
471 h->next = r->headers;
480 static struct http_buf *http_serialize_response(struct http_channel *c,
481 struct http_response *r)
483 struct http_header *h;
485 wrbuf_rewind(c->wrbuf);
486 wrbuf_printf(c->wrbuf, "HTTP/1.1 %s %s\r\n", r->code, r->msg);
487 for (h = r->headers; h; h = h->next)
488 wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
491 wrbuf_printf(c->wrbuf, "Content-length: %d\r\n", r->payload ?
492 (int) strlen(r->payload) : 0);
493 wrbuf_printf(c->wrbuf, "Content-type: text/xml\r\n");
496 xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
503 yaz_log(YLOG_WARN, "Sending non-wellformed "
504 "response (bug #1162");
505 yaz_log(YLOG_WARN, "payload: %s", r->payload);
509 wrbuf_puts(c->wrbuf, "\r\n");
512 wrbuf_puts(c->wrbuf, r->payload);
514 return http_buf_bywrbuf(c->wrbuf);
517 // Serialize a HTTP request
518 static struct http_buf *http_serialize_request(struct http_request *r)
520 struct http_channel *c = r->channel;
521 struct http_header *h;
522 struct http_argument *a;
524 wrbuf_rewind(c->wrbuf);
525 wrbuf_printf(c->wrbuf, "%s %s", r->method, r->path);
529 wrbuf_putc(c->wrbuf, '?');
530 for (a = r->arguments; a; a = a->next) {
531 if (a != r->arguments)
532 wrbuf_putc(c->wrbuf, '&');
533 wrbuf_printf(c->wrbuf, "%s=%s", a->name, a->value);
537 wrbuf_printf(c->wrbuf, " HTTP/%s\r\n", r->http_version);
539 for (h = r->headers; h; h = h->next)
540 wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
542 wrbuf_puts(c->wrbuf, "\r\n");
544 return http_buf_bywrbuf(c->wrbuf);
548 static int http_weshouldproxy(struct http_request *rq)
550 if (proxy_addr && !strstr(rq->path, "search.pz2"))
556 struct http_header * http_header_append(struct http_channel *ch,
557 struct http_header * hp,
561 struct http_header *hpnew = 0;
566 while (hp && hp->next)
569 if(name && strlen(name)&& value && strlen(value)){
570 hpnew = nmem_malloc(ch->nmem, sizeof *hpnew);
571 hpnew->name = nmem_strdup(ch->nmem, name);
572 hpnew->value = nmem_strdup(ch->nmem, value);
586 static int http_proxy(struct http_request *rq)
588 struct http_channel *c = rq->channel;
589 struct http_proxy *p = c->proxy;
590 struct http_header *hp;
591 struct http_buf *requestbuf;
592 char server_via[128] = "";
593 char server_port[16] = "";
594 struct conf_server *ser = global_parameters.server;
596 if (!p) // This is a new connection. Create a proxy channel
603 if (!(pe = getprotobyname("tcp"))) {
606 if ((sock = socket(PF_INET, SOCK_STREAM, pe->p_proto)) < 0)
608 yaz_log(YLOG_WARN|YLOG_ERRNO, "socket");
611 if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)
612 &one, sizeof(one)) < 0)
614 if ((flags = fcntl(sock, F_GETFL, 0)) < 0)
615 yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
616 if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0)
617 yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
618 if (connect(sock, (struct sockaddr *) proxy_addr,
619 sizeof(*proxy_addr)) < 0)
620 if (errno != EINPROGRESS)
622 yaz_log(YLOG_WARN|YLOG_ERRNO, "Proxy connect");
626 p = xmalloc(sizeof(struct http_proxy));
629 p->first_response = 1;
631 // We will add EVENT_OUTPUT below
632 p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT);
633 iochan_setdata(p->iochan, p);
634 pazpar2_add_channel(p->iochan);
637 // Do _not_ modify Host: header, just checking it's existence
638 for (hp = rq->headers; hp; hp = hp->next)
639 if (!strcmp(hp->name, "Host"))
643 yaz_log(YLOG_WARN, "Failed to find Host header in proxy");
647 // Add new header about paraz2 version, host, remote client address, etc.
650 hp = http_header_append(c, hp,
651 "X-Pazpar2-Version", PACKAGE_VERSION);
652 hp = http_header_append(c, hp,
653 "X-Pazpar2-Server-Host", ser->host);
654 sprintf(server_port, "%d", ser->port);
655 hp = http_header_append(c, hp,
656 "X-Pazpar2-Server-Port", server_port);
657 sprintf(server_via, "1.1 %s:%s (%s/%s)",
658 ser->host, server_port, PACKAGE_NAME, PACKAGE_VERSION);
659 hp = http_header_append(c, hp, "Via" , server_via);
660 hp = http_header_append(c, hp, "X-Forwarded-For", c->addr);
663 requestbuf = http_serialize_request(rq);
664 http_buf_enqueue(&p->oqueue, requestbuf);
665 iochan_setflag(p->iochan, EVENT_OUTPUT);
669 void http_send_response(struct http_channel *ch)
671 struct http_response *rs = ch->response;
675 hb = http_serialize_response(ch, rs);
678 yaz_log(YLOG_WARN, "Failed to serialize HTTP response");
679 http_destroy(ch->iochan);
683 http_buf_enqueue(&ch->oqueue, hb);
684 iochan_setflag(ch->iochan, EVENT_OUTPUT);
685 ch->state = Http_Idle;
689 static void http_io(IOCHAN i, int event)
691 struct http_channel *hc = iochan_getdata(i);
696 struct http_buf *htbuf;
699 htbuf = http_buf_create();
700 res = read(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1);
701 if (res == -1 && errno == EAGAIN)
703 http_buf_destroy(htbuf);
708 http_buf_destroy(htbuf);
712 htbuf->buf[res] = '\0';
714 http_buf_enqueue(&hc->iqueue, htbuf);
716 if (hc->state == Http_Busy)
718 if ((reqlen = request_check(hc->iqueue)) <= 2)
721 nmem_reset(hc->nmem);
722 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
724 yaz_log(YLOG_WARN, "Failed to parse request");
729 yaz_log(YLOG_LOG, "Request: %s %s%s%s", hc->request->method,
731 *hc->request->search ? "?" : "",
732 hc->request->search);
733 if (http_weshouldproxy(hc->request))
734 http_proxy(hc->request);
737 // Execute our business logic!
738 hc->state = Http_Busy;
743 yaz_log(YLOG_DEBUG, "We think we have more input to read. Forcing event");
744 iochan_setevent(i, EVENT_INPUT);
752 struct http_buf *wb = hc->oqueue;
753 res = write(iochan_getfd(hc->iochan), wb->buf + wb->offset, wb->len);
756 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
762 hc->oqueue = hc->oqueue->next;
763 http_buf_destroy(wb);
771 if (!strcmp(hc->version, "1.0"))
778 iochan_clearflag(i, EVENT_OUTPUT);
780 iochan_setevent(hc->iochan, EVENT_INPUT);
785 if (!hc->oqueue && hc->proxy && !hc->proxy->iochan)
786 http_destroy(i); // Server closed; we're done
789 yaz_log(YLOG_WARN, "Unexpected event on connection");
795 // If this hostname contains our proxy host as a prefix, replace with myurl
796 static char *sub_hostname(struct http_channel *c, char *buf)
799 if (strlen(buf) > 1023)
801 if (strncmp(buf, "http://", 7))
803 if (!strncmp(buf + 7, proxy_url, strlen(proxy_url)))
806 strcat(tmp, buf + strlen(proxy_url) + 7);
807 return nmem_strdup(c->nmem, tmp);
813 // Handles I/O on a client connection to a backend web server (proxy mode)
814 static void proxy_io(IOCHAN pi, int event)
816 struct http_proxy *pc = iochan_getdata(pi);
817 struct http_channel *hc = pc->channel;
822 struct http_buf *htbuf;
825 htbuf = http_buf_create();
826 res = read(iochan_getfd(pi), htbuf->buf, HTTP_BUF_SIZE -1);
827 if (res == 0 || (res < 0 && errno != EINPROGRESS))
831 yaz_log(YLOG_WARN, "Proxy read came up short");
832 // Close channel and alert client HTTP channel that we're gone
833 http_buf_destroy(htbuf);
834 close(iochan_getfd(pi));
840 http_destroy(hc->iochan);
846 htbuf->buf[res] = '\0';
850 if (pc->first_response) // Check if this is a redirect
853 if ((len = package_check(htbuf->buf)))
855 struct http_response *res = http_parse_response_buf(hc, htbuf->buf, len);
858 struct http_header *h;
859 for (h = res->headers; h; h = h->next)
860 if (!strcmp(h->name, "Location"))
862 // We found a location header. Rewrite it.
863 struct http_buf *buf;
864 h->value = sub_hostname(hc, h->value);
865 buf = http_serialize_response(hc, res);
866 yaz_log(YLOG_LOG, "Proxy rewrite");
867 http_buf_enqueue(&hc->oqueue, buf);
873 pc->first_response = 0;
876 // Write any remaining payload
877 if (htbuf->len - htbuf->offset > 0)
878 http_buf_enqueue(&hc->oqueue, htbuf);
880 iochan_setflag(hc->iochan, EVENT_OUTPUT);
883 if (!(htbuf = pc->oqueue))
885 iochan_clearflag(pi, EVENT_OUTPUT);
888 res = write(iochan_getfd(pi), htbuf->buf + htbuf->offset, htbuf->len);
891 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
892 http_destroy(hc->iochan);
895 if (res == htbuf->len)
897 struct http_buf *np = htbuf->next;
898 http_buf_destroy(htbuf);
904 htbuf->offset += res;
908 iochan_setflags(pi, EVENT_INPUT); // Turns off output flag
912 yaz_log(YLOG_WARN, "Unexpected event on connection");
913 http_destroy(hc->iochan);
917 static void http_fire_observers(struct http_channel *c);
918 static void http_destroy_observers(struct http_channel *c);
921 static void http_destroy(IOCHAN i)
923 struct http_channel *s = iochan_getdata(i);
927 if (s->proxy->iochan)
929 close(iochan_getfd(s->proxy->iochan));
930 iochan_destroy(s->proxy->iochan);
932 http_buf_destroy_queue(s->proxy->oqueue);
935 http_buf_destroy_queue(s->iqueue);
936 http_buf_destroy_queue(s->oqueue);
937 http_fire_observers(s);
938 http_destroy_observers(s);
939 s->next = http_channel_freelist;
940 http_channel_freelist = s;
941 close(iochan_getfd(i));
945 static struct http_channel *http_create(const char *addr)
947 struct http_channel *r = http_channel_freelist;
951 http_channel_freelist = r->next;
953 wrbuf_rewind(r->wrbuf);
957 r = xmalloc(sizeof(struct http_channel));
958 r->nmem = nmem_create();
959 r->wrbuf = wrbuf_alloc();
963 r->iqueue = r->oqueue = 0;
964 r->state = Http_Idle;
969 yaz_log(YLOG_WARN, "Invalid HTTP forward address");
972 strcpy(r->addr, addr);
978 /* Accept a new command connection */
979 static void http_accept(IOCHAN i, int event)
981 struct sockaddr_in addr;
982 int fd = iochan_getfd(i);
987 struct http_channel *ch;
990 if ((s = accept(fd, (struct sockaddr *) &addr, &len)) < 0)
992 yaz_log(YLOG_WARN|YLOG_ERRNO, "accept");
995 if ((flags = fcntl(s, F_GETFL, 0)) < 0)
996 yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
997 if (fcntl(s, F_SETFL, flags | O_NONBLOCK) < 0)
998 yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
1000 yaz_log(YLOG_DEBUG, "New command connection");
1001 c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT);
1003 ch = http_create(inet_ntoa(addr.sin_addr));
1005 iochan_setdata(c, ch);
1007 pazpar2_add_channel(c);
1010 /* Create a http-channel listener, syntax [host:]port */
1011 void http_init(const char *addr)
1016 struct sockaddr_in myaddr;
1021 yaz_log(YLOG_LOG, "HTTP listener %s", addr);
1023 memset(&myaddr, 0, sizeof myaddr);
1024 myaddr.sin_family = AF_INET;
1025 pp = strchr(addr, ':');
1028 int len = pp - addr;
1032 strncpy(hostname, addr, len);
1033 hostname[len] = '\0';
1034 if (!(he = gethostbyname(hostname))){
1035 yaz_log(YLOG_FATAL, "Unable to resolve '%s'", hostname);
1039 memcpy(&myaddr.sin_addr.s_addr, he->h_addr_list[0], he->h_length);
1040 port = atoi(pp + 1);
1042 yaz_log(YLOG_LOG, "HTTP address %s:%d",
1043 "" == he->h_addr_list[0] ? he->h_addr_list[0] : "127.0.0.1" ,
1050 myaddr.sin_addr.s_addr = INADDR_ANY;
1053 myaddr.sin_port = htons(port);
1055 if (!(p = getprotobyname("tcp"))) {
1058 if ((l = socket(PF_INET, SOCK_STREAM, p->p_proto)) < 0)
1059 yaz_log(YLOG_FATAL|YLOG_ERRNO, "socket");
1060 if (setsockopt(l, SOL_SOCKET, SO_REUSEADDR, (char*)
1061 &one, sizeof(one)) < 0)
1064 if (bind(l, (struct sockaddr *) &myaddr, sizeof myaddr) < 0)
1066 yaz_log(YLOG_FATAL|YLOG_ERRNO, "bind");
1069 if (listen(l, SOMAXCONN) < 0)
1071 yaz_log(YLOG_FATAL|YLOG_ERRNO, "listen");
1075 c = iochan_create(l, http_accept, EVENT_INPUT | EVENT_EXCEPT);
1076 pazpar2_add_channel(c);
1079 void http_set_proxyaddr(char *host, char *base_url)
1085 strcpy(myurl, base_url);
1086 strcpy(proxy_url, host);
1087 p = strchr(host, ':');
1088 yaz_log(YLOG_DEBUG, "Proxying for %s", host);
1089 yaz_log(YLOG_LOG, "HTTP backend %s", proxy_url);
1096 if (!(he = gethostbyname(host)))
1098 fprintf(stderr, "Failed to lookup '%s'\n", host);
1101 proxy_addr = xmalloc(sizeof(struct sockaddr_in));
1102 proxy_addr->sin_family = he->h_addrtype;
1103 memcpy(&proxy_addr->sin_addr.s_addr, he->h_addr_list[0], he->h_length);
1104 proxy_addr->sin_port = htons(port);
1107 static void http_fire_observers(struct http_channel *c)
1109 http_channel_observer_t p = c->observers;
1112 p->destroy(p->data, c);
1117 static void http_destroy_observers(struct http_channel *c)
1119 while (c->observers)
1121 http_channel_observer_t obs = c->observers;
1122 c->observers = obs->next;
1127 http_channel_observer_t http_add_observer(struct http_channel *c, void *data,
1128 http_channel_destroy_t des)
1130 http_channel_observer_t obs = xmalloc(sizeof(*obs));
1134 obs->next = c->observers;
1139 void http_remove_observer(http_channel_observer_t obs)
1141 struct http_channel *c = obs->chan;
1142 http_channel_observer_t found, *p = &c->observers;
1151 struct http_channel *http_channel_observer_chan(http_channel_observer_t obs)
1159 * indent-tabs-mode: nil
1161 * vim: shiftwidth=4 tabstop=8 expandtab