src/http.c

   1 /* This file is part of Pazpar2.
   2    Copyright (C) 2006-2011 Index Data
   3
   4 Pazpar2 is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20 #if HAVE_CONFIG_H
  21 #include <config.h>
  22 #endif
  23
  24 #if HAVE_SYS_TIME_H
  25 #include <sys/time.h>
  26 #endif
  27
  28 #include <stdio.h>
  29 #ifdef WIN32
  30 #include <winsock.h>
  31 typedef int socklen_t;
  32 #endif
  33
  34 #if HAVE_SYS_SOCKET_H
  35 #include <sys/socket.h>
  36 #endif
  37
  38 #include <sys/types.h>
  39
  40 #include <yaz/snprintf.h>
  41 #if HAVE_UNISTD_H
  42 #include <unistd.h>
  43 #endif
  44
  45 #include <stdlib.h>
  46 #include <string.h>
  47 #include <ctype.h>
  48 #include <fcntl.h>
  49 #if HAVE_NETDB_H
  50 #include <netdb.h>
  51 #endif
  52
  53 #include <errno.h>
  54 #include <assert.h>
  55 #include <string.h>
  56
  57 #if HAVE_NETINET_IN_H
  58 #include <netinet/in.h>
  59 #endif
  60
  61 #if HAVE_ARPA_INET_H
  62 #include <arpa/inet.h>
  63 #endif
  64
  65 #include <yaz/yaz-util.h>
  66 #include <yaz/comstack.h>
  67 #include <yaz/nmem.h>
  68 #include <yaz/mutex.h>
  69
  70 #include "ppmutex.h"
  71 #include "session.h"
  72 #include "http.h"
  73 #include "parameters.h"
  74
  75 #define MAX_HTTP_HEADER 4096
  76
  77 #ifdef WIN32
  78 #define strncasecmp _strnicmp
  79 #define strcasecmp _stricmp
  80 #endif
  81
  82 struct http_buf
  83 {
  84 #define HTTP_BUF_SIZE 4096
  85     char buf[4096];
  86     int offset;
  87     int len;
  88     struct http_buf *next;
  89 };
  90
  91
  92 static void proxy_io(IOCHAN i, int event);
  93 static struct http_channel *http_channel_create(http_server_t http_server,
  94                                                 const char *addr,
  95                                                 struct conf_server *server);
  96 static void http_channel_destroy(IOCHAN i);
  97 static http_server_t http_server_create(void);
  98 static void http_server_incref(http_server_t hs);
  99
 100 struct http_server
 101 {
 102     YAZ_MUTEX mutex;
 103     int listener_socket;
 104     int ref_count;
 105     http_sessions_t http_sessions;
 106     struct sockaddr_in *proxy_addr;
 107     FILE *record_file;
 108 };
 109
 110 struct http_channel_observer_s {
 111     void *data;
 112     void *data2;
 113     http_channel_destroy_t destroy;
 114     struct http_channel_observer_s *next;
 115     struct http_channel *chan;
 116 };
 117
 118
 119 const char *http_lookup_header(struct http_header *header,
 120                                const char *name)
 121 {
 122     for (; header; header = header->next)
 123         if (!strcasecmp(name, header->name))
 124             return header->value;
 125     return 0;
 126 }
 127
 128 static struct http_buf *http_buf_create(http_server_t hs)
 129 {
 130     struct http_buf *r = xmalloc(sizeof(*r));
 131     r->offset = 0;
 132     r->len = 0;
 133     r->next = 0;
 134     return r;
 135 }
 136
 137 static void http_buf_destroy(http_server_t hs, struct http_buf *b)
 138 {
 139     xfree(b);
 140 }
 141
 142 static void http_buf_destroy_queue(http_server_t hs, struct http_buf *b)
 143 {
 144     struct http_buf *p;
 145     while (b)
 146     {
 147         p = b->next;
 148         http_buf_destroy(hs, b);
 149         b = p;
 150     }
 151 }
 152
 153 static struct http_buf *http_buf_bybuf(http_server_t hs, char *b, int len)
 154 {
 155     struct http_buf *res = 0;
 156     struct http_buf **p = &res;
 157
 158     while (len)
 159     {
 160         int tocopy = len;
 161         if (tocopy > HTTP_BUF_SIZE)
 162             tocopy = HTTP_BUF_SIZE;
 163         *p = http_buf_create(hs);
 164         memcpy((*p)->buf, b, tocopy);
 165         (*p)->len = tocopy;
 166         len -= tocopy;
 167         b += tocopy;
 168         p = &(*p)->next;
 169     }
 170     return res;
 171 }
 172
 173 // Add a (chain of) buffers to the end of an existing queue.
 174 static void http_buf_enqueue(struct http_buf **queue, struct http_buf *b)
 175 {
 176     while (*queue)
 177         queue = &(*queue)->next;
 178     *queue = b;
 179 }
 180
 181 static struct http_buf *http_buf_bywrbuf(http_server_t hs, WRBUF wrbuf)
 182 {
 183     // Heavens to Betsy (buf)!
 184     return http_buf_bybuf(hs, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
 185 }
 186
 187 // Non-destructively collapse chain of buffers into a string (max *len)
 188 // Return
 189 static void http_buf_peek(struct http_buf *b, char *buf, int len)
 190 {
 191     int rd = 0;
 192     while (b && rd < len)
 193     {
 194         int toread = len - rd;
 195         if (toread > b->len)
 196             toread = b->len;
 197         memcpy(buf + rd, b->buf + b->offset, toread);
 198         rd += toread;
 199         b = b->next;
 200     }
 201     buf[rd] = '\0';
 202 }
 203
 204 static int http_buf_size(struct http_buf *b)
 205 {
 206     int sz = 0;
 207     for (; b; b = b->next)
 208         sz += b->len;
 209     return sz;
 210 }
 211
 212 // Ddestructively munch up to len  from head of queue.
 213 static int http_buf_read(http_server_t hs,
 214                          struct http_buf **b, char *buf, int len)
 215 {
 216     int rd = 0;
 217     while ((*b) && rd < len)
 218     {
 219         int toread = len - rd;
 220         if (toread > (*b)->len)
 221             toread = (*b)->len;
 222         memcpy(buf + rd, (*b)->buf + (*b)->offset, toread);
 223         rd += toread;
 224         if (toread < (*b)->len)
 225         {
 226             (*b)->len -= toread;
 227             (*b)->offset += toread;
 228             break;
 229         }
 230         else
 231         {
 232             struct http_buf *n = (*b)->next;
 233             http_buf_destroy(hs, *b);
 234             *b = n;
 235         }
 236     }
 237     buf[rd] = '\0';
 238     return rd;
 239 }
 240
 241 // Buffers may overlap.
 242 static void urldecode(char *i, char *o)
 243 {
 244     while (*i)
 245     {
 246         if (*i == '+')
 247         {
 248             *(o++) = ' ';
 249             i++;
 250         }
 251         else if (*i == '%' && i[1] && i[2])
 252         {
 253             int v;
 254             i++;
 255             sscanf(i, "%2x", &v);
 256             *o++ = v;
 257             i += 2;
 258         }
 259         else
 260             *(o++) = *(i++);
 261     }
 262     *o = '\0';
 263 }
 264
 265 // Warning: Buffers may not overlap
 266 void urlencode(const char *i, char *o)
 267 {
 268     while (*i)
 269     {
 270         if (strchr(" /:", *i))
 271         {
 272             sprintf(o, "%%%.2X", (int) *i);
 273             o += 3;
 274         }
 275         else
 276             *(o++) = *i;
 277         i++;
 278     }
 279     *o = '\0';
 280 }
 281
 282 void http_addheader(struct http_response *r, const char *name, const char *value)
 283 {
 284     struct http_channel *c = r->channel;
 285     struct http_header *h = nmem_malloc(c->nmem, sizeof *h);
 286     h->name = nmem_strdup(c->nmem, name);
 287     h->value = nmem_strdup(c->nmem, value);
 288     h->next = r->headers;
 289     r->headers = h;
 290 }
 291
 292 const char *http_argbyname(struct http_request *r, const char *name)
 293 {
 294     struct http_argument *p;
 295     if (!name)
 296         return 0;
 297     for (p = r->arguments; p; p = p->next)
 298         if (!strcmp(p->name, name))
 299             return p->value;
 300     return 0;
 301 }
 302
 303 const char *http_headerbyname(struct http_header *h, const char *name)
 304 {
 305     for (; h; h = h->next)
 306         if (!strcmp(h->name, name))
 307             return h->value;
 308     return 0;
 309 }
 310
 311 struct http_response *http_create_response(struct http_channel *c)
 312 {
 313     struct http_response *r = nmem_malloc(c->nmem, sizeof(*r));
 314     strcpy(r->code, "200");
 315     r->msg = "OK";
 316     r->channel = c;
 317     r->headers = 0;
 318     r->payload = 0;
 319     r->content_type = "text/xml";
 320     return r;
 321 }
 322
 323
 324 static const char *next_crlf(const char *cp, size_t *skipped)
 325 {
 326     const char *next_cp = strchr(cp, '\n');
 327     if (next_cp)
 328     {
 329         if (next_cp > cp && next_cp[-1] == '\r')
 330             *skipped = next_cp - cp - 1;
 331         else
 332             *skipped = next_cp - cp;
 333         next_cp++;
 334     }
 335     return next_cp;
 336 }
 337
 338 // Check if buf contains a package (minus payload)
 339 static int package_check(const char *buf, int sz)
 340 {
 341     int content_len = 0;
 342     int len = 0;
 343
 344     while (*buf)
 345     {
 346         size_t skipped = 0;
 347         const char *b = next_crlf(buf, &skipped);
 348
 349         if (!b)
 350         {
 351             // we did not find CRLF.. See if buffer is too large..
 352             if (sz >= MAX_HTTP_HEADER-1)
 353                 return MAX_HTTP_HEADER-1; // yes. Return that (will fail later)
 354             break;
 355         }
 356         len += (b - buf);
 357         if (skipped == 0)
 358         {
 359             // CRLF CRLF , i.e. end of header
 360             if (len + content_len <= sz)
 361                 return len + content_len;
 362             break;
 363         }
 364         buf = b;
 365         // following first skip of \r\n so that we don't consider Method
 366         if (!strncasecmp(buf, "Content-Length:", 15))
 367         {
 368             const char *cp = buf+15;
 369             while (*cp == ' ')
 370                 cp++;
 371             content_len = 0;
 372             while (*cp && isdigit(*(const unsigned char *)cp))
 373                 content_len = content_len*10 + (*cp++ - '0');
 374             if (content_len < 0) /* prevent negative offsets */
 375                 content_len = 0;
 376         }
 377     }
 378     return 0;     // incomplete request
 379 }
 380
 381 // Check if we have a request. Return 0 or length
 382 static int request_check(struct http_buf *queue)
 383 {
 384     char tmp[MAX_HTTP_HEADER];
 385
 386     // only peek at the header..
 387     http_buf_peek(queue, tmp, MAX_HTTP_HEADER-1);
 388     // still we only return non-zero if the complete request is received..
 389     return package_check(tmp, http_buf_size(queue));
 390 }
 391
 392 struct http_response *http_parse_response_buf(struct http_channel *c, const char *buf, int len)
 393 {
 394     char tmp[MAX_HTTP_HEADER];
 395     struct http_response *r = http_create_response(c);
 396     char *p, *p2;
 397     struct http_header **hp = &r->headers;
 398
 399     if (len >= MAX_HTTP_HEADER)
 400         return 0;
 401     memcpy(tmp, buf, len);
 402     for (p = tmp; *p && *p != ' '; p++) // Skip HTTP version
 403         ;
 404     p++;
 405     // Response code
 406     for (p2 = p; *p2 && *p2 != ' ' && p2 - p < 3; p2++)
 407         r->code[p2 - p] = *p2;
 408     if (!(p = strstr(tmp, "\r\n")))
 409         return 0;
 410     p += 2;
 411     while (*p)
 412     {
 413         if (!(p2 = strstr(p, "\r\n")))
 414             return 0;
 415         if (p == p2) // End of headers
 416             break;
 417         else
 418         {
 419             struct http_header *h = *hp = nmem_malloc(c->nmem, sizeof(*h));
 420             char *value = strchr(p, ':');
 421             if (!value)
 422                 return 0;
 423             *(value++) = '\0';
 424             h->name = nmem_strdup(c->nmem, p);
 425             while (isspace(*(const unsigned char *) value))
 426                 value++;
 427             if (value >= p2)  // Empty header;
 428             {
 429                 h->value = "";
 430                 p = p2 + 2;
 431                 continue;
 432             }
 433             *p2 = '\0';
 434             h->value = nmem_strdup(c->nmem, value);
 435             h->next = 0;
 436             hp = &h->next;
 437             p = p2 + 2;
 438         }
 439     }
 440     return r;
 441 }
 442
 443 static int http_parse_arguments(struct http_request *r, NMEM nmem,
 444                                 const char *args)
 445 {
 446     const char *p2 = args;
 447
 448     while (*p2)
 449     {
 450         struct http_argument *a;
 451         const char *equal = strchr(p2, '=');
 452         const char *eoa = strchr(p2, '&');
 453         if (!equal)
 454         {
 455             yaz_log(YLOG_WARN, "Expected '=' in argument");
 456             return -1;
 457         }
 458         if (!eoa)
 459             eoa = equal + strlen(equal); // last argument
 460         else if (equal > eoa)
 461         {
 462             yaz_log(YLOG_WARN, "Missing '&' in argument");
 463             return -1;
 464         }
 465         a = nmem_malloc(nmem, sizeof(struct http_argument));
 466         a->name = nmem_strdupn(nmem, p2, equal - p2);
 467         a->value = nmem_strdupn(nmem, equal+1, eoa - equal - 1);
 468         urldecode(a->name, a->name);
 469         urldecode(a->value, a->value);
 470         a->next = r->arguments;
 471         r->arguments = a;
 472         p2 = eoa;
 473         while (*p2 == '&')
 474             p2++;
 475     }
 476     return 0;
 477 }
 478
 479 struct http_request *http_parse_request(struct http_channel *c,
 480                                         struct http_buf **queue,
 481                                         int len)
 482 {
 483     struct http_request *r = nmem_malloc(c->nmem, sizeof(*r));
 484     char *p, *p2;
 485     char *start = nmem_malloc(c->nmem, len+1);
 486     char *buf = start;
 487
 488     if (http_buf_read(c->http_server, queue, buf, len) < len)
 489     {
 490         yaz_log(YLOG_WARN, "http_buf_read < len (%d)", len);
 491         return 0;
 492     }
 493     r->search = "";
 494     r->channel = c;
 495     r->arguments = 0;
 496     r->headers = 0;
 497     r->content_buf = 0;
 498     r->content_len = 0;
 499     // Parse first line
 500     for (p = buf, p2 = r->method; *p && *p != ' ' && p - buf < 19; p++)
 501         *(p2++) = *p;
 502     if (*p != ' ')
 503     {
 504         yaz_log(YLOG_WARN, "Unexpected HTTP method in request");
 505         return 0;
 506     }
 507     *p2 = '\0';
 508
 509     if (!(buf = strchr(buf, ' ')))
 510     {
 511         yaz_log(YLOG_WARN, "Missing Request-URI in HTTP request");
 512         return 0;
 513     }
 514     buf++;
 515     if (!(p = strchr(buf, ' ')))
 516     {
 517         yaz_log(YLOG_WARN, "HTTP Request-URI not terminated (too long?)");
 518         return 0;
 519     }
 520     *(p++) = '\0';
 521     if ((p2 = strchr(buf, '?'))) // Do we have arguments?
 522         *(p2++) = '\0';
 523     r->path = nmem_strdup(c->nmem, buf);
 524     if (p2)
 525     {
 526         r->search = nmem_strdup(c->nmem, p2);
 527         // Parse Arguments
 528         http_parse_arguments(r, c->nmem, p2);
 529     }
 530     buf = p;
 531
 532     if (strncmp(buf, "HTTP/", 5))
 533         strcpy(r->http_version, "1.0");
 534     else
 535     {
 536         size_t skipped;
 537         buf += 5; // strlen("HTTP/")
 538
 539         p = (char*) next_crlf(buf, &skipped);
 540         if (!p || skipped < 3 || skipped > 5)
 541             return 0;
 542
 543         memcpy(r->http_version, buf, skipped);
 544         r->http_version[skipped] = '\0';
 545         buf = p;
 546     }
 547     strcpy(c->version, r->http_version);
 548
 549     r->headers = 0;
 550     while (*buf)
 551     {
 552         size_t skipped;
 553
 554         p = (char *) next_crlf(buf, &skipped);
 555         if (!p)
 556         {
 557             return 0;
 558         }
 559         else if (skipped == 0)
 560         {
 561             buf = p;
 562             break;
 563         }
 564         else
 565         {
 566             char *cp;
 567             char *n_v = nmem_malloc(c->nmem, skipped+1);
 568             struct http_header *h = nmem_malloc(c->nmem, sizeof(*h));
 569
 570             memcpy(n_v, buf, skipped);
 571             n_v[skipped] = '\0';
 572
 573             if (!(cp = strchr(n_v, ':')))
 574                 return 0;
 575             h->name = nmem_strdupn(c->nmem, n_v, cp - n_v);
 576             cp++;
 577             while (isspace(*cp))
 578                 cp++;
 579             h->value = nmem_strdup(c->nmem, cp);
 580             h->next = r->headers;
 581             r->headers = h;
 582             buf = p;
 583         }
 584     }
 585
 586     // determine if we do keep alive
 587     if (!strcmp(c->version, "1.0"))
 588     {
 589         const char *v = http_lookup_header(r->headers, "Connection");
 590         if (v && !strcmp(v, "Keep-Alive"))
 591             c->keep_alive = 1;
 592         else
 593             c->keep_alive = 0;
 594     }
 595     else
 596     {
 597         const char *v = http_lookup_header(r->headers, "Connection");
 598         if (v && !strcmp(v, "close"))
 599             c->keep_alive = 0;
 600         else
 601             c->keep_alive = 1;
 602     }
 603     if (buf < start + len)
 604     {
 605         const char *content_type = http_lookup_header(r->headers,
 606                                                       "Content-Type");
 607         r->content_len = start + len - buf;
 608         r->content_buf = buf;
 609
 610         if (!yaz_strcmp_del("application/x-www-form-urlencoded",
 611                             content_type, "; "))
 612         {
 613             http_parse_arguments(r, c->nmem, r->content_buf);
 614         }
 615     }
 616     return r;
 617 }
 618
 619 static struct http_buf *http_serialize_response(struct http_channel *c,
 620         struct http_response *r)
 621 {
 622     struct http_header *h;
 623
 624     wrbuf_rewind(c->wrbuf);
 625     wrbuf_printf(c->wrbuf, "HTTP/%s %s %s\r\n", c->version, r->code, r->msg);
 626     for (h = r->headers; h; h = h->next)
 627         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
 628     if (r->payload)
 629     {
 630         wrbuf_printf(c->wrbuf, "Content-Length: %d\r\n", r->payload ?
 631                 (int) strlen(r->payload) : 0);
 632         wrbuf_printf(c->wrbuf, "Content-Type: %s\r\n", r->content_type);
 633         if (!strcmp(r->content_type, "text/xml"))
 634         {
 635             xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
 636             if (doc)
 637             {
 638                 xmlFreeDoc(doc);
 639             }
 640             else
 641             {
 642                 yaz_log(YLOG_WARN, "Sending non-wellformed "
 643                         "response (bug #1162");
 644                 yaz_log(YLOG_WARN, "payload: %s", r->payload);
 645             }
 646         }
 647     }
 648     wrbuf_puts(c->wrbuf, "\r\n");
 649
 650     if (r->payload)
 651         wrbuf_puts(c->wrbuf, r->payload);
 652
 653     if (global_parameters.dump_records > 1)
 654     {
 655         FILE *lf = yaz_log_file();
 656         yaz_log(YLOG_LOG, "Response:");
 657         fwrite(wrbuf_buf(c->wrbuf), 1, wrbuf_len(c->wrbuf), lf);
 658     }
 659     return http_buf_bywrbuf(c->http_server, c->wrbuf);
 660 }
 661
 662 // Serialize a HTTP request
 663 static struct http_buf *http_serialize_request(struct http_request *r)
 664 {
 665     struct http_channel *c = r->channel;
 666     struct http_header *h;
 667
 668     wrbuf_rewind(c->wrbuf);
 669     wrbuf_printf(c->wrbuf, "%s %s%s%s", r->method, r->path,
 670                  *r->search ? "?" : "", r->search);
 671
 672     wrbuf_printf(c->wrbuf, " HTTP/%s\r\n", r->http_version);
 673
 674     for (h = r->headers; h; h = h->next)
 675         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
 676
 677     wrbuf_puts(c->wrbuf, "\r\n");
 678
 679     if (r->content_buf)
 680         wrbuf_write(c->wrbuf, r->content_buf, r->content_len);
 681
 682 #if 0
 683     yaz_log(YLOG_LOG, "WRITING TO PROXY:\n%s\n----",
 684             wrbuf_cstr(c->wrbuf));
 685 #endif
 686     return http_buf_bywrbuf(c->http_server, c->wrbuf);
 687 }
 688
 689
 690 static int http_weshouldproxy(struct http_request *rq)
 691 {
 692     struct http_channel *c = rq->channel;
 693     if (c->server->http_server->proxy_addr && !strstr(rq->path, "search.pz2"))
 694         return 1;
 695     return 0;
 696 }
 697
 698
 699 struct http_header * http_header_append(struct http_channel *ch,
 700                                         struct http_header * hp,
 701                                         const char *name,
 702                                         const char *value)
 703 {
 704     struct http_header *hpnew = 0;
 705
 706     if (!hp | !ch)
 707         return 0;
 708
 709     while (hp && hp->next)
 710         hp = hp->next;
 711
 712     if(name && strlen(name)&& value && strlen(value)){
 713         hpnew = nmem_malloc(ch->nmem, sizeof *hpnew);
 714         hpnew->name = nmem_strdup(ch->nmem, name);
 715         hpnew->value = nmem_strdup(ch->nmem, value);
 716
 717         hpnew->next = 0;
 718         hp->next = hpnew;
 719         hp = hp->next;
 720
 721         return hpnew;
 722     }
 723
 724     return hp;
 725 }
 726
 727
 728 static int is_inprogress(void)
 729 {
 730 #ifdef WIN32
 731     if (WSAGetLastError() == WSAEWOULDBLOCK)
 732         return 1;
 733 #else
 734     if (errno == EINPROGRESS)
 735         return 1;
 736 #endif
 737     return 0;
 738 }
 739
 740 static void enable_nonblock(int sock)
 741 {
 742     int flags;
 743 #ifdef WIN32
 744     flags = (flags & CS_FLAGS_BLOCKING) ? 0 : 1;
 745     if (ioctlsocket(sock, FIONBIO, &flags) < 0)
 746         yaz_log(YLOG_FATAL|YLOG_ERRNO, "ioctlsocket");
 747 #else
 748     if ((flags = fcntl(sock, F_GETFL, 0)) < 0)
 749         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
 750     if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0)
 751         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
 752 #endif
 753 }
 754
 755 static int http_proxy(struct http_request *rq)
 756 {
 757     struct http_channel *c = rq->channel;
 758     struct http_proxy *p = c->proxy;
 759     struct http_header *hp;
 760     struct http_buf *requestbuf;
 761     char server_port[16] = "";
 762     struct conf_server *ser = c->server;
 763
 764     if (!p) // This is a new connection. Create a proxy channel
 765     {
 766         int sock;
 767         struct protoent *pe;
 768         int one = 1;
 769
 770         if (!(pe = getprotobyname("tcp"))) {
 771             abort();
 772         }
 773         if ((sock = socket(PF_INET, SOCK_STREAM, pe->p_proto)) < 0)
 774         {
 775             yaz_log(YLOG_WARN|YLOG_ERRNO, "socket");
 776             return -1;
 777         }
 778         if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)
 779                         &one, sizeof(one)) < 0)
 780             abort();
 781         enable_nonblock(sock);
 782         if (connect(sock, (struct sockaddr *)
 783                     c->server->http_server->proxy_addr,
 784                     sizeof(*c->server->http_server->proxy_addr)) < 0)
 785         {
 786             if (!is_inprogress())
 787             {
 788                 yaz_log(YLOG_WARN|YLOG_ERRNO, "Proxy connect");
 789                 return -1;
 790             }
 791         }
 792         p = xmalloc(sizeof(struct http_proxy));
 793         p->oqueue = 0;
 794         p->channel = c;
 795         p->first_response = 1;
 796         c->proxy = p;
 797         // We will add EVENT_OUTPUT below
 798         p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT, "http_proxy");
 799         iochan_setdata(p->iochan, p);
 800
 801         iochan_add(ser->iochan_man, p->iochan);
 802     }
 803
 804     // Do _not_ modify Host: header, just checking it's existence
 805
 806     if (!http_lookup_header(rq->headers, "Host"))
 807     {
 808         yaz_log(YLOG_WARN, "Failed to find Host header in proxy");
 809         return -1;
 810     }
 811
 812     // Add new header about paraz2 version, host, remote client address, etc.
 813     {
 814         char server_via[128];
 815
 816         hp = rq->headers;
 817         hp = http_header_append(c, hp,
 818                                 "X-Pazpar2-Version", PACKAGE_VERSION);
 819         hp = http_header_append(c, hp,
 820                                 "X-Pazpar2-Server-Host", ser->host);
 821         sprintf(server_port, "%d",  ser->port);
 822         hp = http_header_append(c, hp,
 823                                 "X-Pazpar2-Server-Port", server_port);
 824         yaz_snprintf(server_via, sizeof(server_via),
 825                      "1.1 %s:%s (%s/%s)",
 826                      ser->host ? ser->host : "@",
 827                      server_port, PACKAGE_NAME, PACKAGE_VERSION);
 828         hp = http_header_append(c, hp, "Via" , server_via);
 829         hp = http_header_append(c, hp, "X-Forwarded-For", c->addr);
 830     }
 831
 832     requestbuf = http_serialize_request(rq);
 833
 834     http_buf_enqueue(&p->oqueue, requestbuf);
 835     iochan_setflag(p->iochan, EVENT_OUTPUT);
 836     return 0;
 837 }
 838
 839 void http_send_response(struct http_channel *ch)
 840 {
 841     struct http_response *rs = ch->response;
 842     struct http_buf *hb;
 843
 844     assert(rs);
 845     hb = http_serialize_response(ch, rs);
 846     if (!hb)
 847     {
 848         yaz_log(YLOG_WARN, "Failed to serialize HTTP response");
 849         http_channel_destroy(ch->iochan);
 850     }
 851     else
 852     {
 853         http_buf_enqueue(&ch->oqueue, hb);
 854         iochan_setflag(ch->iochan, EVENT_OUTPUT);
 855         ch->state = Http_Idle;
 856     }
 857 }
 858
 859 static void http_error(struct http_channel *hc, int no, const char *msg)
 860 {
 861     struct http_response *rs = http_create_response(hc);
 862
 863     hc->response = rs;
 864     hc->keep_alive = 0;  // not keeping this HTTP session alive
 865
 866     sprintf(rs->code, "%d", no);
 867
 868     rs->msg = nmem_strdup(hc->nmem, msg);
 869     rs->payload = nmem_malloc(hc->nmem, 100);
 870     yaz_snprintf(rs->payload, 99, "<error>HTTP Error %d: %s</error>\n",
 871                  no, msg);
 872     http_send_response(hc);
 873 }
 874
 875 static void http_io(IOCHAN i, int event)
 876 {
 877     struct http_channel *hc = iochan_getdata(i);
 878     while (event)
 879     {
 880         if (event == EVENT_INPUT)
 881         {
 882             int res, reqlen;
 883             struct http_buf *htbuf;
 884
 885             htbuf = http_buf_create(hc->http_server);
 886             res = recv(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1, 0);
 887             if (res == -1 && errno == EAGAIN)
 888             {
 889                 http_buf_destroy(hc->http_server, htbuf);
 890                 return;
 891             }
 892             if (res <= 0)
 893             {
 894 #if HAVE_SYS_TIME_H
 895                 if (hc->http_server->record_file)
 896                 {
 897                     struct timeval tv;
 898                     gettimeofday(&tv, 0);
 899                     fprintf(hc->http_server->record_file, "r %lld %lld %lld 0\n",
 900                             (long long) tv.tv_sec, (long long) tv.tv_usec,
 901                             (long long) iochan_getfd(i));
 902                 }
 903 #endif
 904                 http_buf_destroy(hc->http_server, htbuf);
 905                 fflush(hc->http_server->record_file);
 906                 http_channel_destroy(i);
 907                 return;
 908             }
 909             htbuf->buf[res] = '\0';
 910             htbuf->len = res;
 911             http_buf_enqueue(&hc->iqueue, htbuf);
 912
 913             while (1)
 914             {
 915                 if (hc->state == Http_Busy)
 916                     return;
 917                 reqlen = request_check(hc->iqueue);
 918                 if (reqlen <= 2)
 919                     return;
 920                 // we have a complete HTTP request
 921                 nmem_reset(hc->nmem);
 922 #if HAVE_SYS_TIME_H
 923                 if (hc->http_server->record_file)
 924                 {
 925                     struct timeval tv;
 926                     int sz = 0;
 927                     struct http_buf *hb;
 928                     for (hb = hc->iqueue; hb; hb = hb->next)
 929                         sz += hb->len;
 930                     gettimeofday(&tv, 0);
 931                     fprintf(hc->http_server->record_file, "r %lld %lld %lld %d\n",
 932                             (long long) tv.tv_sec, (long long) tv.tv_usec,
 933                             (long long) iochan_getfd(i), sz);
 934                     for (hb = hc->iqueue; hb; hb = hb->next)
 935                         fwrite(hb->buf, 1, hb->len, hc->http_server->record_file);
 936                     fflush(hc->http_server->record_file);
 937                 }
 938  #endif
 939                 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
 940                 {
 941                     yaz_log(YLOG_WARN, "Failed to parse request");
 942                     http_error(hc, 400, "Bad Request");
 943                     return;
 944                 }
 945                 hc->response = 0;
 946                 yaz_log(YLOG_LOG, "Request: %s %s%s%s", hc->request->method,
 947                         hc->request->path,
 948                         *hc->request->search ? "?" : "",
 949                         hc->request->search);
 950                 if (hc->request->content_buf)
 951                     yaz_log(YLOG_LOG, "%s", hc->request->content_buf);
 952                 if (http_weshouldproxy(hc->request))
 953                     http_proxy(hc->request);
 954                 else
 955                 {
 956                     // Execute our business logic!
 957                     hc->state = Http_Busy;
 958                     http_command(hc);
 959                 }
 960             }
 961         }
 962         else if (event == EVENT_OUTPUT)
 963         {
 964             event = 0;
 965             if (hc->oqueue)
 966             {
 967                 struct http_buf *wb = hc->oqueue;
 968                 int res;
 969                 res = send(iochan_getfd(hc->iochan),
 970                            wb->buf + wb->offset, wb->len, 0);
 971                 if (res <= 0)
 972                 {
 973                     yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
 974                     http_channel_destroy(i);
 975                     return;
 976                 }
 977                 if (res == wb->len)
 978                 {
 979 #if HAVE_SYS_TIME_H
 980                     if (hc->http_server->record_file)
 981                     {
 982                         struct timeval tv;
 983                         int sz = wb->offset + wb->len;
 984                         gettimeofday(&tv, 0);
 985                         fprintf(hc->http_server->record_file, "w %lld %lld %lld %d\n",
 986                                 (long long) tv.tv_sec, (long long) tv.tv_usec,
 987                                 (long long) iochan_getfd(i), sz);
 988                         fwrite(wb->buf, 1, wb->offset + wb->len,
 989                                hc->http_server->record_file);
 990                         fputc('\n', hc->http_server->record_file);
 991                         fflush(hc->http_server->record_file);
 992                     }
 993  #endif
 994                     hc->oqueue = hc->oqueue->next;
 995                     http_buf_destroy(hc->http_server, wb);
 996                 }
 997                 else
 998                 {
 999                     wb->len -= res;
1000                     wb->offset += res;
1001                 }
1002                 if (!hc->oqueue)
1003                 {
1004                     if (!hc->keep_alive)
1005                     {
1006                         http_channel_destroy(i);
1007                         return;
1008                     }
1009                     else
1010                     {
1011                         iochan_clearflag(i, EVENT_OUTPUT);
1012                         if (hc->iqueue)
1013                             event = EVENT_INPUT;
1014                     }
1015                 }
1016             }
1017             if (!hc->oqueue && hc->proxy && !hc->proxy->iochan)
1018                 http_channel_destroy(i); // Server closed; we're done
1019         }
1020         else
1021         {
1022             yaz_log(YLOG_WARN, "Unexpected event on connection");
1023             http_channel_destroy(i);
1024             event = 0;
1025         }
1026     }
1027 }
1028
1029 // Handles I/O on a client connection to a backend web server (proxy mode)
1030 static void proxy_io(IOCHAN pi, int event)
1031 {
1032     struct http_proxy *pc = iochan_getdata(pi);
1033     struct http_channel *hc = pc->channel;
1034
1035     switch (event)
1036     {
1037         int res;
1038         struct http_buf *htbuf;
1039
1040         case EVENT_INPUT:
1041             htbuf = http_buf_create(hc->http_server);
1042             res = recv(iochan_getfd(pi), htbuf->buf, HTTP_BUF_SIZE -1, 0);
1043             if (res == 0 || (res < 0 && !is_inprogress()))
1044             {
1045                 if (hc->oqueue)
1046                 {
1047                     yaz_log(YLOG_WARN, "Proxy read came up short");
1048                     // Close channel and alert client HTTP channel that we're gone
1049                     http_buf_destroy(hc->http_server, htbuf);
1050 #ifdef WIN32
1051                     closesocket(iochan_getfd(pi));
1052 #else
1053                     close(iochan_getfd(pi));
1054 #endif
1055                     iochan_destroy(pi);
1056                     pc->iochan = 0;
1057                 }
1058                 else
1059                 {
1060                     http_channel_destroy(hc->iochan);
1061                     return;
1062                 }
1063             }
1064             else
1065             {
1066                 htbuf->buf[res] = '\0';
1067                 htbuf->offset = 0;
1068                 htbuf->len = res;
1069                 // Write any remaining payload
1070                 if (htbuf->len - htbuf->offset > 0)
1071                     http_buf_enqueue(&hc->oqueue, htbuf);
1072             }
1073             iochan_setflag(hc->iochan, EVENT_OUTPUT);
1074             break;
1075         case EVENT_OUTPUT:
1076             if (!(htbuf = pc->oqueue))
1077             {
1078                 iochan_clearflag(pi, EVENT_OUTPUT);
1079                 return;
1080             }
1081             res = send(iochan_getfd(pi), htbuf->buf + htbuf->offset, htbuf->len, 0);
1082             if (res <= 0)
1083             {
1084                 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
1085                 http_channel_destroy(hc->iochan);
1086                 return;
1087             }
1088             if (res == htbuf->len)
1089             {
1090                 struct http_buf *np = htbuf->next;
1091                 http_buf_destroy(hc->http_server, htbuf);
1092                 pc->oqueue = np;
1093             }
1094             else
1095             {
1096                 htbuf->len -= res;
1097                 htbuf->offset += res;
1098             }
1099
1100             if (!pc->oqueue) {
1101                 iochan_setflags(pi, EVENT_INPUT); // Turns off output flag
1102             }
1103             break;
1104         default:
1105             yaz_log(YLOG_WARN, "Unexpected event on connection");
1106             http_channel_destroy(hc->iochan);
1107             break;
1108     }
1109 }
1110
1111 static void http_fire_observers(struct http_channel *c);
1112 static void http_destroy_observers(struct http_channel *c);
1113
1114 // Cleanup channel
1115 static void http_channel_destroy(IOCHAN i)
1116 {
1117     struct http_channel *s = iochan_getdata(i);
1118     http_server_t http_server;
1119
1120     if (s->proxy)
1121     {
1122         if (s->proxy->iochan)
1123         {
1124 #ifdef WIN32
1125             closesocket(iochan_getfd(s->proxy->iochan));
1126 #else
1127             close(iochan_getfd(s->proxy->iochan));
1128 #endif
1129             iochan_destroy(s->proxy->iochan);
1130         }
1131         http_buf_destroy_queue(s->http_server, s->proxy->oqueue);
1132         xfree(s->proxy);
1133     }
1134     http_buf_destroy_queue(s->http_server, s->iqueue);
1135     http_buf_destroy_queue(s->http_server, s->oqueue);
1136     http_fire_observers(s);
1137     http_destroy_observers(s);
1138
1139     http_server = s->http_server; /* save it for destroy (decref) */
1140
1141     http_server_destroy(http_server);
1142
1143 #ifdef WIN32
1144     closesocket(iochan_getfd(i));
1145 #else
1146     close(iochan_getfd(i));
1147 #endif
1148     iochan_destroy(i);
1149     nmem_destroy(s->nmem);
1150     wrbuf_destroy(s->wrbuf);
1151     xfree(s);
1152 }
1153
1154 static struct http_channel *http_channel_create(http_server_t hs,
1155                                                 const char *addr,
1156                                                 struct conf_server *server)
1157 {
1158     struct http_channel *r;
1159
1160     r = xmalloc(sizeof(struct http_channel));
1161     r->nmem = nmem_create();
1162     r->wrbuf = wrbuf_alloc();
1163
1164     http_server_incref(hs);
1165     r->http_server = hs;
1166     r->http_sessions = hs->http_sessions;
1167     assert(r->http_sessions);
1168     r->server = server;
1169     r->proxy = 0;
1170     r->iochan = 0;
1171     r->iqueue = r->oqueue = 0;
1172     r->state = Http_Idle;
1173     r->keep_alive = 0;
1174     r->request = 0;
1175     r->response = 0;
1176     if (!addr)
1177     {
1178         yaz_log(YLOG_WARN, "Invalid HTTP forward address");
1179         exit(1);
1180     }
1181     strcpy(r->addr, addr);
1182     r->observers = 0;
1183     return r;
1184 }
1185
1186
1187 /* Accept a new command connection */
1188 static void http_accept(IOCHAN i, int event)
1189 {
1190     struct sockaddr_in addr;
1191     int fd = iochan_getfd(i);
1192     socklen_t len;
1193     int s;
1194     IOCHAN c;
1195     struct http_channel *ch;
1196     struct conf_server *server = iochan_getdata(i);
1197
1198     len = sizeof addr;
1199     if ((s = accept(fd, (struct sockaddr *) &addr, &len)) < 0)
1200     {
1201         yaz_log(YLOG_WARN|YLOG_ERRNO, "accept");
1202         return;
1203     }
1204     enable_nonblock(s);
1205
1206     yaz_log(YLOG_DEBUG, "New command connection");
1207     c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT, "http_session_socket");
1208
1209     ch = http_channel_create(server->http_server, inet_ntoa(addr.sin_addr),
1210                              server);
1211     ch->iochan = c;
1212     iochan_setdata(c, ch);
1213     iochan_add(server->iochan_man, c);
1214 }
1215
1216 /* Create a http-channel listener, syntax [host:]port */
1217 int http_init(const char *addr, struct conf_server *server,
1218               const char *record_fname)
1219 {
1220     IOCHAN c;
1221     int l;
1222     struct protoent *p;
1223     struct sockaddr_in myaddr;
1224     int one = 1;
1225     const char *pp;
1226     short port;
1227     FILE *record_file = 0;
1228
1229     yaz_log(YLOG_LOG, "HTTP listener %s", addr);
1230
1231
1232     if (record_fname)
1233     {
1234         record_file = fopen(record_fname, "wb");
1235         if (!record_file)
1236         {
1237             yaz_log(YLOG_FATAL|YLOG_ERRNO, "fopen %s", record_fname);
1238             return 1;
1239         }
1240     }
1241
1242     memset(&myaddr, 0, sizeof myaddr);
1243     myaddr.sin_family = AF_INET;
1244     pp = strchr(addr, ':');
1245     if (pp)
1246     {
1247         WRBUF w = wrbuf_alloc();
1248         struct hostent *he;
1249
1250         wrbuf_write(w, addr, pp - addr);
1251         wrbuf_puts(w, "");
1252
1253         he = gethostbyname(wrbuf_cstr(w));
1254         wrbuf_destroy(w);
1255         if (!he)
1256         {
1257             yaz_log(YLOG_FATAL, "Unable to resolve '%s'", addr);
1258             return 1;
1259         }
1260         memcpy(&myaddr.sin_addr.s_addr, he->h_addr_list[0], he->h_length);
1261         port = atoi(pp + 1);
1262     }
1263     else
1264     {
1265         port = atoi(addr);
1266         myaddr.sin_addr.s_addr = INADDR_ANY;
1267     }
1268
1269     myaddr.sin_port = htons(port);
1270
1271     if (!(p = getprotobyname("tcp"))) {
1272         return 1;
1273     }
1274     if ((l = socket(PF_INET, SOCK_STREAM, p->p_proto)) < 0)
1275         yaz_log(YLOG_FATAL|YLOG_ERRNO, "socket");
1276     if (setsockopt(l, SOL_SOCKET, SO_REUSEADDR, (char*)
1277                     &one, sizeof(one)) < 0)
1278         return 1;
1279
1280     if (bind(l, (struct sockaddr *) &myaddr, sizeof myaddr) < 0)
1281     {
1282         yaz_log(YLOG_FATAL|YLOG_ERRNO, "bind");
1283         return 1;
1284     }
1285     if (listen(l, SOMAXCONN) < 0)
1286     {
1287         yaz_log(YLOG_FATAL|YLOG_ERRNO, "listen");
1288         return 1;
1289     }
1290
1291     server->http_server = http_server_create();
1292
1293     server->http_server->record_file = record_file;
1294     server->http_server->listener_socket = l;
1295
1296     c = iochan_create(l, http_accept, EVENT_INPUT | EVENT_EXCEPT, "http_server");
1297     iochan_setdata(c, server);
1298
1299     iochan_add(server->iochan_man, c);
1300     return 0;
1301 }
1302
1303 void http_close_server(struct conf_server *server)
1304 {
1305     /* break the event_loop (select) by closing down the HTTP listener sock */
1306     if (server->http_server->listener_socket)
1307     {
1308 #ifdef WIN32
1309         closesocket(server->http_server->listener_socket);
1310 #else
1311         close(server->http_server->listener_socket);
1312 #endif
1313     }
1314 }
1315
1316 void http_set_proxyaddr(const char *host, struct conf_server *server)
1317 {
1318     const char *p;
1319     short port;
1320     struct hostent *he;
1321     WRBUF w = wrbuf_alloc();
1322
1323     yaz_log(YLOG_LOG, "HTTP backend  %s", host);
1324
1325     p = strchr(host, ':');
1326     if (p)
1327     {
1328         port = atoi(p + 1);
1329         wrbuf_write(w, host, p - host);
1330         wrbuf_puts(w, "");
1331     }
1332     else
1333     {
1334         port = 80;
1335         wrbuf_puts(w, host);
1336     }
1337     if (!(he = gethostbyname(wrbuf_cstr(w))))
1338     {
1339         fprintf(stderr, "Failed to lookup '%s'\n", wrbuf_cstr(w));
1340         exit(1);
1341     }
1342     wrbuf_destroy(w);
1343
1344     server->http_server->proxy_addr = xmalloc(sizeof(struct sockaddr_in));
1345     server->http_server->proxy_addr->sin_family = he->h_addrtype;
1346     memcpy(&server->http_server->proxy_addr->sin_addr.s_addr,
1347            he->h_addr_list[0], he->h_length);
1348     server->http_server->proxy_addr->sin_port = htons(port);
1349 }
1350
1351 static void http_fire_observers(struct http_channel *c)
1352 {
1353     http_channel_observer_t p = c->observers;
1354     while (p)
1355     {
1356         p->destroy(p->data, c, p->data2);
1357         p = p->next;
1358     }
1359 }
1360
1361 static void http_destroy_observers(struct http_channel *c)
1362 {
1363     while (c->observers)
1364     {
1365         http_channel_observer_t obs = c->observers;
1366         c->observers = obs->next;
1367         xfree(obs);
1368     }
1369 }
1370
1371 http_channel_observer_t http_add_observer(struct http_channel *c, void *data,
1372                                           http_channel_destroy_t des)
1373 {
1374     http_channel_observer_t obs = xmalloc(sizeof(*obs));
1375     obs->chan = c;
1376     obs->data = data;
1377     obs->data2 = 0;
1378     obs->destroy= des;
1379     obs->next = c->observers;
1380     c->observers = obs;
1381     return obs;
1382 }
1383
1384 void http_remove_observer(http_channel_observer_t obs)
1385 {
1386     struct http_channel *c = obs->chan;
1387     http_channel_observer_t found, *p = &c->observers;
1388     while (*p != obs)
1389         p = &(*p)->next;
1390     found = *p;
1391     assert(found);
1392     *p = (*p)->next;
1393     xfree(found);
1394 }
1395
1396 struct http_channel *http_channel_observer_chan(http_channel_observer_t obs)
1397 {
1398     return obs->chan;
1399 }
1400
1401 void http_observer_set_data2(http_channel_observer_t obs, void *data2)
1402 {
1403     obs->data2 = data2;
1404 }
1405
1406 http_server_t http_server_create(void)
1407 {
1408     http_server_t hs = xmalloc(sizeof(*hs));
1409     hs->mutex = 0;
1410     hs->proxy_addr = 0;
1411     hs->ref_count = 1;
1412     hs->http_sessions = 0;
1413
1414     hs->record_file = 0;
1415     return hs;
1416 }
1417
1418 void http_server_destroy(http_server_t hs)
1419 {
1420     if (hs)
1421     {
1422         int r;
1423
1424         yaz_mutex_enter(hs->mutex); /* OK: hs->mutex may be NULL */
1425         r = --(hs->ref_count);
1426         yaz_mutex_leave(hs->mutex);
1427
1428         if (r == 0)
1429         {
1430             http_sessions_destroy(hs->http_sessions);
1431             xfree(hs->proxy_addr);
1432             yaz_mutex_destroy(&hs->mutex);
1433             if (hs->record_file)
1434                 fclose(hs->record_file);
1435             xfree(hs);
1436         }
1437     }
1438 }
1439
1440 void http_server_incref(http_server_t hs)
1441 {
1442     assert(hs);
1443     yaz_mutex_enter(hs->mutex);
1444     (hs->ref_count)++;
1445     yaz_mutex_leave(hs->mutex);
1446 }
1447
1448 void http_mutex_init(struct conf_server *server)
1449 {
1450     assert(server);
1451
1452     assert(server->http_server->mutex == 0);
1453     pazpar2_mutex_create(&server->http_server->mutex, "http_server");
1454     server->http_server->http_sessions = http_sessions_create();
1455 }
1456
1457 /*
1458  * Local variables:
1459  * c-basic-offset: 4
1460  * c-file-style: "Stroustrup"
1461  * indent-tabs-mode: nil
1462  * End:
1463  * vim: shiftwidth=4 tabstop=8 expandtab
1464  */
1465