Extensive rewrite of the HTTP code. The new code is significantly cleaner and

more robust, and somewhat more efficient. It also handles authorization and
redirects properly, and supports timeouts like the FTP code.

Many thanks to Umemoto-san for his assistance with IPv6 support, both here
and in other parts of libfetch.

1 files changed, 685 insertions, 446 deletions

diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index d8ae744..dfe1314 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 2000 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -12,7 +12,7 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
- *    derived from this software without specific prior written permission
+ *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
@@ -25,46 +25,15 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD$
+ *      $FreeBSD$
  */
 
-/*
- * The base64 code in this file is based on code from MIT fetch, which
- * has the following copyright and license:
- *
- *-
- * Copyright 1997 Massachusetts Institute of Technology
- *
- * Permission to use, copy, modify, and distribute this software and
- * its documentation for any purpose and without fee is hereby
- * granted, provided that both the above copyright notice and this
- * permission notice appear in all copies, that both the above
- * copyright notice and this permission notice appear in all
- * supporting documentation, and that the name of M.I.T. not be used
- * in advertising or publicity pertaining to distribution of the
- * software without specific, written prior permission.	 M.I.T. makes
- * no representations about the suitability of this software for any
- * purpose.  It is provided "as is" without express or implied
- * warranty.
- * 
- * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
- * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
- * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE. */
-
 #include <sys/param.h>
 #include <sys/socket.h>
 
-#include <err.h>
 #include <ctype.h>
+#include <err.h>
+#include <errno.h>
 #include <locale.h>
 #include <netdb.h>
 #include <stdarg.h>
@@ -78,148 +47,402 @@
 #include "common.h"
 #include "httperr.h"
 
-extern char *__progname;
+extern char *__progname; /* XXX not portable */
 
-#define ENDL "\r\n"
+/* Maximum number of redirects to follow */
+#define MAX_REDIRECT 5
 
-#define HTTP_OK		200
-#define HTTP_PARTIAL	206
-#define HTTP_MOVED	302
+/* Symbolic names for reply codes we care about */
+#define HTTP_OK			200
+#define HTTP_PARTIAL		206
+#define HTTP_MOVED_PERM		301
+#define HTTP_MOVED_TEMP		302
+#define HTTP_SEE_OTHER		303
+#define HTTP_NEED_AUTH		401
+#define HTTP_NEED_PROXY_AUTH	403
+#define HTTP_PROTOCOL_ERROR	999
+
+#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
+                            || (xyz) == HTTP_MOVED_TEMP \
+                            || (xyz) == HTTP_SEE_OTHER)
+
+
+
+/*****************************************************************************
+ * I/O functions for decoding chunked streams
+ */
 
 struct cookie
 {
-    FILE *real_f;
-#define ENC_NONE 0
-#define ENC_CHUNKED 1
-    int encoding;			/* 1 = chunked, 0 = none */
-#define HTTPCTYPELEN 59
-    char content_type[HTTPCTYPELEN+1];
-    char *buf;
-    int b_cur, eof;
-    unsigned b_len, chunksize;
+    int		 fd;
+    char	*buf;
+    size_t	 b_size;
+    size_t	 b_len;
+    int		 b_pos;
+    int		 eof;
+    int		 error;
+    long	 chunksize;
+#ifdef DEBUG
+    long	 total;
+#endif
 };
 
 /*
- * Send a formatted line; optionally echo to terminal
+ * Get next chunk header
  */
 static int
-_http_cmd(FILE *f, char *fmt, ...)
+_http_new_chunk(struct cookie *c)
 {
-    va_list ap;
-
-    va_start(ap, fmt);
-    vfprintf(f, fmt, ap);
-    DEBUG(fprintf(stderr, "\033[1m>>> "));
-    DEBUG(vfprintf(stderr, fmt, ap));
-    DEBUG(fprintf(stderr, "\033[m"));
-    va_end(ap);
+    char *p;
+    
+    if (_fetch_getln(c->fd, &c->buf, &c->b_size, &c->b_len) == -1)
+	return -1;
+    
+    if (c->b_len < 2 || !ishexnumber(*c->buf))
+	return -1;
+    
+    for (p = c->buf; !isspace(*p) && *p != ';' && p < c->buf + c->b_len; ++p)
+	if (!ishexnumber(*p))
+	    return -1;
+	else if (isdigit(*p))
+	    c->chunksize = c->chunksize * 16 + *p - '0';
+	else
+	    c->chunksize = c->chunksize * 16 + 10 + tolower(*p) - 'a';
+    
+#ifdef DEBUG
+    c->total += c->chunksize;
+    if (c->chunksize == 0)
+	fprintf(stderr, "\033[1m_http_fillbuf(): "
+		"end of last chunk\033[m\n");
+    else
+	fprintf(stderr, "\033[1m_http_fillbuf(): "
+		"new chunk: %ld (%ld)\033[m\n", c->chunksize, c->total);
+#endif
     
-    return 0; /* XXX */
+    return c->chunksize;
 }
 
 /*
  * Fill the input buffer, do chunk decoding on the fly
  */
-static char *
+static int
 _http_fillbuf(struct cookie *c)
 {
-    char *ln;
-    unsigned int len;
-
+    if (c->error)
+	return -1;
     if (c->eof)
-	return NULL;
-
-    if (c->encoding == ENC_NONE) {
-	c->buf = fgetln(c->real_f, &(c->b_len));
-	c->b_cur = 0;
-    } else if (c->encoding == ENC_CHUNKED) {
-	if (c->chunksize == 0) {
-	    ln = fgetln(c->real_f, &len);
-	    if (len <= 2)
-		return NULL;
-	    DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): new chunk: "
-			  "%*.*s\033[m\n", (int)len-2, (int)len-2, ln));
-	    sscanf(ln, "%x", &(c->chunksize));
-	    if (!c->chunksize) {
-		DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): "
-			      "end of last chunk\033[m\n"));
-		c->eof = 1;
-		return NULL;
-	    }
-	    DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): "
-			  "new chunk: %X\033[m\n", c->chunksize));
+	return 0;
+    
+    if (c->chunksize == 0) {
+	switch (_http_new_chunk(c)) {
+	case -1:
+	    c->error = 1;
+	    return -1;
+	case 0:
+	    c->eof = 1;
+	    return 0;
 	}
-	c->buf = fgetln(c->real_f, &(c->b_len));
-	if (c->b_len > c->chunksize)
-	    c->b_len = c->chunksize;
-	c->chunksize -= c->b_len;
-	c->b_cur = 0;
     }
-    else return NULL; /* unknown encoding */
-    return c->buf;
+
+    if (c->b_size < c->chunksize) {
+	char *tmp;
+
+	if ((tmp = realloc(c->buf, c->chunksize)) == NULL)
+	    return -1;
+	c->buf = tmp;
+	c->b_size = c->chunksize;
+    }
+    
+    if ((c->b_len = read(c->fd, c->buf, c->chunksize)) == -1)
+	return -1;
+    c->chunksize -= c->b_len;
+    
+    if (c->chunksize == 0) {
+	char endl[2];
+	read(c->fd, endl, 2);
+    }
+    
+    c->b_pos = 0;
+    
+    return c->b_len;
 }
 
 /*
  * Read function
  */
 static int
-_http_readfn(struct cookie *c, char *buf, int len)
+_http_readfn(void *v, char *buf, int len)
 {
-    int l, pos = 0;
-    while (len) {
+    struct cookie *c = (struct cookie *)v;
+    int l, pos;
+
+    if (c->error)
+	return -1;
+    if (c->eof)
+	return 0;
+
+    for (pos = 0; len > 0; pos += l, len -= l) {
 	/* empty buffer */
-	if (!c->buf || (c->b_cur == c->b_len))
-	    if (!_http_fillbuf(c))
+	if (!c->buf || c->b_pos == c->b_len)
+	    if (_http_fillbuf(c) < 1)
 		break;
-
-	l = c->b_len - c->b_cur;
-	if (len < l) l = len;
-	memcpy(buf + pos, c->buf + c->b_cur, l);
-	c->b_cur += l;
-	pos += l;
-	len -= l;
+	l = c->b_len - c->b_pos;
+	if (len < l)
+	    l = len;
+	bcopy(c->buf + c->b_pos, buf + pos, l);
+	c->b_pos += l;
     }
-    
-    if (ferror(c->real_f))
+
+    if (!pos && c->error)
 	return -1;
-    else return pos;
+    return pos;
 }
 
 /*
  * Write function
  */
 static int
-_http_writefn(struct cookie *c, const char *buf, int len)
+_http_writefn(void *v, const char *buf, int len)
 {
-    size_t r = fwrite(buf, 1, (size_t)len, c->real_f);
-    return r ? r : -1;
+    struct cookie *c = (struct cookie *)v;
+    
+    return write(c->fd, buf, len);
 }
 
 /*
  * Close function
  */
 static int
-_http_closefn(struct cookie *c)
+_http_closefn(void *v)
 {
-    int r = fclose(c->real_f);
+    struct cookie *c = (struct cookie *)v;
+    int r;
+
+    r = close(c->fd);
+    if (c->buf)
+	free(c->buf);
     free(c);
-    return (r == EOF) ? -1 : 0;
+    return r;
+}
+
+/*
+ * Wrap a file descriptor up
+ */
+static FILE *
+_http_funopen(int fd)
+{
+    struct cookie *c;
+    FILE *f;
+
+    if ((c = calloc(1, sizeof *c)) == NULL) {
+	_fetch_syserr();
+	return NULL;
+    }
+    c->fd = fd;
+    if (!(f = funopen(c, _http_readfn, _http_writefn, NULL, _http_closefn))) {
+	_fetch_syserr();
+	free(c);
+	return NULL;
+    }
+    return f;
+}
+
+
+/*****************************************************************************
+ * Helper functions for talking to the server and parsing its replies
+ */
+
+/* Header types */
+typedef enum {
+    hdr_syserror = -2,
+    hdr_error = -1,
+    hdr_end = 0,
+    hdr_unknown = 1,
+    hdr_content_length,
+    hdr_content_range,
+    hdr_last_modified,
+    hdr_location,
+    hdr_transfer_encoding
+} hdr;
+
+/* Names of interesting headers */
+static struct {
+    hdr		 num;
+    char	*name;
+} hdr_names[] = {
+    { hdr_content_length,	"Content-Length" },
+    { hdr_content_range,	"Content-Range" },
+    { hdr_last_modified,	"Last-Modified" },
+    { hdr_location,		"Location" },
+    { hdr_transfer_encoding,	"Transfer-Encoding" },
+    { hdr_unknown,		NULL },
+};
+
+static char	*reply_buf;
+static size_t	 reply_size;
+static size_t	 reply_length;
+
+/*
+ * Send a formatted line; optionally echo to terminal
+ */
+static int
+_http_cmd(int fd, char *fmt, ...)
+{
+    va_list ap;
+    size_t len;
+    char *msg;
+    int r;
+
+    va_start(ap, fmt);
+    len = vasprintf(&msg, fmt, ap);
+    va_end(ap);
+    
+    if (msg == NULL) {
+	errno = ENOMEM;
+	_fetch_syserr();
+	return -1;
+    }
+    
+    r = _fetch_putln(fd, msg, len);
+    free(msg);
+    
+    if (r == -1) {
+	_fetch_syserr();
+	return -1;
+    }
+    
+    return 0;
+}
+
+/*
+ * Get and parse status line
+ */
+static int
+_http_get_reply(int fd)
+{
+    if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1)
+	return -1;
+    /*
+     * A valid status line looks like "HTTP/m.n xyz reason" where m
+     * and n are the major and minor protocol version numbers and xyz
+     * is the reply code.
+     * We grok HTTP 1.0 and 1.1, so m must be 1 and n must be 0 or 1.
+     * We don't care about the reason phrase.
+     */
+    if (strncmp(reply_buf, "HTTP/1.", 7) != 0
+	|| (reply_buf[7] != '0' && reply_buf[7] != '1') || reply_buf[8] != ' '
+	|| !isdigit(reply_buf[9])
+	|| !isdigit(reply_buf[10])
+	|| !isdigit(reply_buf[11]))
+	return HTTP_PROTOCOL_ERROR;
+    
+    return ((reply_buf[9] - '0') * 100
+	    + (reply_buf[10] - '0') * 10
+	    + (reply_buf[11] - '0'));
 }
 
 /*
- * Extract content type from cookie
+ * Check a header; if the type matches the given string, return a
+ * pointer to the beginning of the value.
  */
-char *
-fetchContentType(FILE *f)
+static char *
+_http_match(char *str, char *hdr)
 {
+    while (*str && *hdr && tolower(*str++) == tolower(*hdr++))
+	/* nothing */;
+    if (*str || *hdr != ':')
+	return NULL;
+    while (*hdr && isspace(*++hdr))
+	/* nothing */;
+    return hdr;
+}
+
+/*
+ * Get the next header and return the appropriate symbolic code.
+ */
+static hdr
+_http_next_header(int fd, char **p)
+{
+    int i;
+    
+    if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1)
+	return hdr_syserror;
+    while (reply_length && isspace(reply_buf[reply_length-1]))
+	reply_length--;
+    reply_buf[reply_length] = 0;
+    if (reply_length == 0)
+	return hdr_end;
     /*
-     * We have no way of making sure this really *is* one of our cookies,
-     * so just check for a null pointer and hope for the best.
+     * We could check for malformed headers but we don't really care.
+     * A valid header starts with a token immediately followed by a
+     * colon; a token is any sequence of non-control, non-whitespace
+     * characters except "()<>@,;:\\\"{}".
      */
-    return f->_cookie ? (((struct cookie *)f->_cookie)->content_type) : NULL;
+    for (i = 0; hdr_names[i].num != hdr_unknown; i++)
+	if ((*p = _http_match(hdr_names[i].name, reply_buf)) != NULL)
+	    return hdr_names[i].num;
+    return hdr_unknown;
+}
+
+/*
+ * Parse a last-modified header
+ */
+static time_t
+_http_parse_mtime(char *p)
+{
+    char locale[64];
+    struct tm tm;
+
+    strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale);
+    setlocale(LC_TIME, "C");
+    strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+    /* XXX should add support for date-2 and date-3 */
+    setlocale(LC_TIME, locale);
+    DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d "
+		  "%02d:%02d:%02d\033[m]\n",
+		  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+		  tm.tm_hour, tm.tm_min, tm.tm_sec));
+    return timegm(&tm);
 }
 
 /*
+ * Parse a content-length header
+ */
+static off_t
+_http_parse_length(char *p)
+{
+    off_t len;
+    
+    for (len = 0; *p && isdigit(*p); ++p)
+	len = len * 10 + (*p - '0');
+    DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", len));
+    return len;
+}
+
+/*
+ * Parse a content-range header
+ */
+static off_t
+_http_parse_range(char *p)
+{
+    off_t off;
+    
+    if (strncasecmp(p, "bytes ", 6) != 0)
+	return -1;
+    for (p += 6, off = 0; *p && isdigit(*p); ++p)
+	off = off * 10 + *p - '0';
+    if (*p != '-')
+	return -1;
+    DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", off));
+    return off;
+}
+
+
+/*****************************************************************************
+ * Helper functions for authorization
+ */
+
+/*
  * Base64 encoding
  */
 static char *
@@ -279,7 +502,7 @@ _http_base64(char *src)
  * Encode username and password
  */
 static int
-_http_basic_auth(FILE *f, char *hdr, char *usr, char *pwd)
+_http_basic_auth(int fd, char *hdr, char *usr, char *pwd)
 {
     char *upw, *auth;
     int r;
@@ -290,7 +513,7 @@ _http_basic_auth(FILE *f, char *hdr, char *usr, char *pwd)
     free(upw);
     if (auth == NULL)
 	return -1;
-    r = _http_cmd(f, "%s: Basic %s" ENDL, hdr, auth);
+    r = _http_cmd(fd, "%s: Basic %s", hdr, auth);
     free(auth);
     return r;
 }
@@ -299,7 +522,7 @@ _http_basic_auth(FILE *f, char *hdr, char *usr, char *pwd)
  * Send an authorization header
  */
 static int
-_http_authorize(FILE *f, char *hdr, char *p)
+_http_authorize(int fd, char *hdr, char *p)
 {
     /* basic authorization */
     if (strncasecmp(p, "basic:", 6) == 0) {
@@ -316,40 +539,138 @@ _http_authorize(FILE *f, char *hdr, char *p)
 	user = str;
 	pwd = strchr(str, ':');
 	*pwd++ = '\0';
-	r = _http_basic_auth(f, hdr, user, pwd);
+	r = _http_basic_auth(fd, hdr, user, pwd);
 	free(str);
 	return r;
     }
     return -1;
 }
 
+
+/*****************************************************************************
+ * Helper functions for connecting to a server or proxy
+ */
+
 /*
- * Connect to server or proxy
+ * Connect to the specified HTTP proxy server.
  */
-static FILE *
-_http_connect(struct url *URL, char *flags, int *proxy)
+static int
+_http_proxy_connect(char *proxy, int af, int verbose)
 {
-    int direct, sd = -1, verbose;
+    char *hostname, *p;
+    int fd, port;
+
+    /* get hostname */
+    hostname = NULL;
 #ifdef INET6
-    int af = AF_UNSPEC;
-#else
-    int af = AF_INET;
+    /* host part can be an IPv6 address enclosed in square brackets */
+    if (*proxy == '[') {
+	if ((p = strchr(proxy, ']')) == NULL) {
+	    /* no terminating bracket */
+	    /* XXX should set an error code */
+	    goto ouch;
+	}
+	if (p[1] != '\0' && p[1] != ':') {
+	    /* garbage after address */
+	    /* XXX should set an error code */
+	    goto ouch;
+	}
+	if ((hostname = malloc(p - proxy)) == NULL) {
+	    errno = ENOMEM;
+	    _fetch_syserr();
+	    goto ouch;
+	}
+	strncpy(hostname, proxy + 1, p - proxy - 1);
+	hostname[p - proxy - 1] = '\0';
+	++p;
+    } else {
+#endif /* INET6 */
+	if ((p = strchr(proxy, ':')) == NULL)
+	    p = strchr(proxy, '\0');
+	if ((hostname = malloc(p - proxy + 1)) == NULL) {
+	    errno = ENOMEM;
+	    _fetch_syserr();
+	    goto ouch;
+	}
+	strncpy(hostname, proxy, p - proxy);
+	hostname[p - proxy] = '\0';
+#ifdef INET6
+    }
+#endif /* INET6 */
+    DEBUG(fprintf(stderr, "proxy name: [%s]\n", hostname));
+    
+    /* get port number */
+    port = 0;
+    if (*p == ':') {
+	++p;
+	if (strspn(p, "0123456789") != strlen(p) || strlen(p) > 5) {
+	    /* port number is non-numeric or too long */
+	    /* XXX should set an error code */
+	    goto ouch;
+	}
+	port = atoi(p);
+	if (port < 1 || port > 65535) {
+	    /* port number is out of range */
+	    /* XXX should set an error code */
+	    goto ouch;
+	}
+    }
+    
+    if (!port) {
+#if 0
+	/*
+	 * commented out, since there is currently no service name
+	 * for HTTP proxies
+	 */
+	struct servent *se;
+	
+	if ((se = getservbyname("xxxx", "tcp")) != NULL)
+	    port = ntohs(se->s_port);
+	else
 #endif
-    size_t len;
-    char *px;
-    FILE *f;
+	    port = 3128;
+    }
+    DEBUG(fprintf(stderr, "proxy port: %d\n", port));
+	
+    /* connect */
+    if ((fd = _fetch_connect(hostname, port, af, verbose)) == -1)
+	_fetch_syserr();
+    return fd;
     
+ ouch:
+    if (hostname)
+	free(hostname);
+    return -1;
+}
+
+/*
+ * Connect to the correct HTTP server or proxy. 
+ */
+static int
+_http_connect(struct url *URL, int *proxy, char *flags)
+{
+    int direct, verbose;
+    int af, fd;
+    char *p;
+    
+#ifdef INET6
+    af = AF_UNSPEC;
+#else
+    af = AF_INET;
+#endif
+
     direct = (flags && strchr(flags, 'd'));
     verbose = (flags && strchr(flags, 'v'));
-    if ((flags && strchr(flags, '4')))
+    if (flags && strchr(flags, '4'))
 	af = AF_INET;
-    else if ((flags && strchr(flags, '6')))
+    else if (flags && strchr(flags, '6'))
 	af = AF_INET6;
     
     /* check port */
     if (!URL->port) {
 	struct servent *se;
 
+	/* Scheme can be ftp if we're using a proxy */
 	if (strcasecmp(URL->scheme, "ftp") == 0)
 	    if ((se = getservbyname("ftp", "tcp")) != NULL)
 		URL->port = ntohs(se->s_port);
@@ -362,166 +683,254 @@ _http_connect(struct url *URL, char *flags, int *proxy)
 		URL->port = 80;
     }
     
-    /* attempt to connect to proxy server */
-    if (!direct && (px = getenv("HTTP_PROXY")) != NULL) {
-	char host[MAXHOSTNAMELEN];
-	int port = 0;
+    if (!direct && (p = getenv("HTTP_PROXY")) != NULL) {
+	/* attempt to connect to proxy server */
+	if ((fd = _http_proxy_connect(p, af, verbose)) == -1)
+	    return -1;
+	*proxy = 1;
+    } else {
+	/* if no proxy is configured, try direct */
+	if (strcasecmp(URL->scheme, "ftp") == 0) {
+	    /* can't talk http to an ftp server */
+	    /* XXX should set an error code */
+	    return -1;
+	}
+	if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1)
+	    /* _fetch_connect() has already set an error code */
+	    return -1;
+	*proxy = 0;
+    }
 
-	/* measure length */
+    return fd;
+}
+
+
+/*****************************************************************************
+ * Core
+ */
+
+/*
+ * Send a request and process the reply
+ */
+static FILE *
+_http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
+{
+    struct url *url, *new;
+    int chunked, need_auth, noredirect, proxy, verbose;
+    int code, fd, i, n;
+    off_t offset;
+    char *p;
+    FILE *f;
+    hdr h;
+    char *host;
 #ifdef INET6
-	if (px[0] != '[' ||
-	    (len = strcspn(px, "]")) >= strlen(px) ||
-	    (px[++len] != '\0' && px[len] != ':'))
+    char hbuf[MAXHOSTNAMELEN + 1];
 #endif
-	    len = strcspn(px, ":");
 
-	/* get port (XXX atoi is a little too tolerant perhaps?) */
-	if (px[len] == ':') {
-	    if (strspn(px+len+1, "0123456789") != strlen(px+len+1)
-		|| strlen(px+len+1) > 5) {
-		/* XXX we should emit some kind of warning */
-	    }
-	    port = atoi(px+len+1);
-	    if (port < 1 || port > 65535) {
-		/* XXX we should emit some kind of warning */
+    noredirect = (flags && strchr(flags, 'A'));
+    verbose = (flags && strchr(flags, 'v'));
+
+    n = noredirect ? 1 : MAX_REDIRECT;
+
+    us->size = -1;
+    us->atime = us->mtime = 0;
+    chunked = 0;
+    offset = 0;
+    fd = -1;
+    code = HTTP_PROTOCOL_ERROR; /* just to appease a compiler warning */
+    
+    for (url = URL, i = 0; i < n; ++i) {
+	need_auth = 0;
+    retry:
+	/* connect to server or proxy */
+	if ((fd = _http_connect(url, &proxy, flags)) == -1)
+	    goto ouch;
+
+	host = url->host;
+#ifdef INET6
+	if (strchr(url->host, ':')) {
+	    snprintf(hbuf, sizeof(hbuf), "[%s]", url->host);
+	    host = hbuf;
+	}
+#endif
+
+	/* send request */
+	if (verbose)
+	    _fetch_info("requesting %s://%s:%d%s",
+			url->scheme, host, url->port, url->doc);
+	if (proxy) {
+	    _http_cmd(fd, "%s %s://%s:%d%s HTTP/1.1",
+		      op, url->scheme, host, url->port, url->doc);
+	} else {
+	    _http_cmd(fd, "%s %s HTTP/1.1",
+		      op, url->doc);
+	}
+
+	/* proxy authorization */
+	if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL)
+	    _http_authorize(fd, "Proxy-Authorization", p);
+	
+	/* server authorization */
+	if (need_auth) {
+	    if (*url->user || *url->pwd)
+		_http_basic_auth(fd, "Authorization",
+				 url->user ? url->user : "",
+				 url->pwd ? url->pwd : "");
+	    else if ((p = getenv("HTTP_AUTH")) != NULL)
+		_http_authorize(fd, "Authorization", p);
+	    else {
+		_http_seterr(HTTP_NEED_AUTH);
+		goto ouch;
 	    }
 	}
-	if (!port) {
-#if 0
+
+	/* other headers */
+	_http_cmd(fd, "Host: %s:%d", host, url->port);
+	_http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname);
+	if (URL->offset)
+	    _http_cmd(fd, "Range: bytes=%lld-", url->offset);
+	_http_cmd(fd, "Connection: close");
+	_http_cmd(fd, "");
+
+	/* get reply */
+	switch ((code = _http_get_reply(fd))) {
+	case HTTP_OK:
+	case HTTP_PARTIAL:
+	    /* fine */
+	    break;
+	case HTTP_MOVED_PERM:
+	case HTTP_MOVED_TEMP:
 	    /*
-	     * commented out, since there is currently no service name
-	     * for HTTP proxies
+	     * Not so fine, but we still have to read the headers to
+	     * get the new location.
 	     */
-	    struct servent *se;
-	    
-	    if ((se = getservbyname("xxxx", "tcp")) != NULL)
-		port = ntohs(se->s_port);
-	    else
-#endif
-		port = 3128;
+	    break;
+	case HTTP_NEED_AUTH:
+	    if (need_auth) {
+		/*
+		 * We already sent out authorization code, so there's
+		 * nothing more we can do.
+		 */
+		_http_seterr(code);
+		goto ouch;
+	    }
+	    /* try again, but send the password this time */
+	    if (verbose)
+		_fetch_info("server requires authorization");
+	    need_auth = 1;
+	    close(fd);
+	    goto retry;
+	case HTTP_NEED_PROXY_AUTH:
+	    /*
+	     * If we're talking to a proxy, we already sent our proxy
+	     * authorization code, so there's nothing more we can do.
+	     */
+	    _http_seterr(code);
+	    goto ouch;
+	case HTTP_PROTOCOL_ERROR:
+	    /* fall through */
+	case -1:
+	    _fetch_syserr();
+	    goto ouch;
+	default:
+	    _http_seterr(code);
+	    goto ouch;
 	}
 	
-	/* get host name */
-#ifdef INET6
-	if (len > 1 && px[0] == '[' && px[len - 1] == ']') {
-	    px++;
-	    len -= 2;
-	}
-#endif
-	if (len >= MAXHOSTNAMELEN)
-	    len = MAXHOSTNAMELEN - 1;
-	strncpy(host, px, len);
-	host[len] = 0;
+	/* get headers */
+	do {
+	    switch ((h = _http_next_header(fd, &p))) {
+	    case hdr_syserror:
+		_fetch_syserr();
+		goto ouch;
+	    case hdr_error:
+		_http_seterr(HTTP_PROTOCOL_ERROR);
+		goto ouch;
+	    case hdr_content_length:
+		us->size = _http_parse_length(p);
+		break;
+	    case hdr_content_range:
+		offset = _http_parse_range(p);
+		break;
+	    case hdr_last_modified:
+		us->atime = us->mtime = _http_parse_mtime(p);
+		break;
+	    case hdr_location:
+		if (!HTTP_REDIRECT(code))
+		    break;
+		if (verbose)
+		    _fetch_info("%d redirect to %s", code, p);
+		if ((new = fetchParseURL(p)) == NULL)
+		    /* invalid location */
+		    goto ouch;
+		if (!*new->user && !*new->pwd) {
+		    strcpy(new->user, url->user);
+		    strcpy(new->pwd, url->pwd);
+		}
+		new->offset = url->offset;
+		new->length = url->length;
+		close(fd);
+		us->size = -1;
+		us->atime = us->mtime = 0;
+		chunked = 0;
+		offset = 0;
+		fd = -1;
+		if (url != URL)
+		    fetchFreeURL(url);
+		url = new;
+		continue;
+	    case hdr_transfer_encoding:	
+		/* XXX weak test*/
+		chunked = (strcasecmp(p, "chunked") == 0);
+		break;
+	    case hdr_end:
+		/* fall through */
+	    case hdr_unknown:
+		/* ignore */
+		break;
+	    }
+	} while (h > hdr_end);
 
-	/* connect */
-	sd = _fetch_connect(host, port, af, verbose);
+	if (code == HTTP_OK || code == HTTP_PARTIAL)
+	    break;
     }
 
-    /* if no proxy is configured or could be contacted, try direct */
-    *proxy = (sd != -1);
-    if (!*proxy) {
-	if (strcasecmp(URL->scheme, "ftp") == 0)
-	    goto ouch;
-	if ((sd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1)
-	    goto ouch;
+    /* no success */
+    if (fd == -1) {
+	_http_seterr(code);
+	goto ouch;
     }
 
-    /* reopen as stream */
-    if ((f = fdopen(sd, "r+")) == NULL)
+    /* wrap it up in a FILE */
+    if ((f = chunked ? _http_funopen(fd) : fdopen(fd, "r")) == NULL) {
+	_fetch_syserr();
 	goto ouch;
+    }
+
+    while (offset++ < url->offset)
+	if (fgetc(f) == EOF) {
+	    _fetch_syserr();
+	    fclose(f);
+	    f = NULL;
+	}
+    
+    if (url != URL)
+	fetchFreeURL(url);
     
     return f;
 
-ouch:
-    if (sd >= 0)
-	close(sd);
-    _http_seterr(999); /* XXX do this properly RSN */
+ ouch:
+    if (url != URL)
+	fetchFreeURL(url);
+    if (fd != -1)
+	close(fd);
     return NULL;
 }
 
-/*
- * Check a header line
- */
-static char *
-_http_match(char *str, char *hdr)
-{
-    while (*str && *hdr && tolower(*str++) == tolower(*hdr++))
-	/* nothing */;
-    if (*str || *hdr != ':')
-	return NULL;
-    while (*hdr && isspace(*++hdr))
-	/* nothing */;
-    return hdr;
-}
-
-/*
- * Send a HEAD or GET request
+
+/*****************************************************************************
+ * Entry points
  */
-static int
-_http_request(FILE *f, char *op, struct url *URL, char *flags, int proxy)
-{
-    int e, verbose;
-    char *ln, *p;
-    size_t len;
-    char *host;
-#ifdef INET6
-    char hbuf[MAXHOSTNAMELEN + 1];
-#endif
-    
-    verbose = (flags && strchr(flags, 'v'));
-
-    host = URL->host;
-#ifdef INET6
-    if (strchr(URL->host, ':')) {
-	snprintf(hbuf, sizeof(hbuf), "[%s]", URL->host);
-	host = hbuf;
-    }
-#endif
-    
-    /* send request (proxies require absolute form) */
-    if (verbose)
-	_fetch_info("requesting %s://%s:%d%s",
-		    URL->scheme, host, URL->port, URL->doc);
-    if (proxy)
-	_http_cmd(f, "%s %s://%s:%d%s HTTP/1.1" ENDL,
-		  op, URL->scheme, host, URL->port, URL->doc);
-    else
-	_http_cmd(f, "%s %s HTTP/1.1" ENDL, op, URL->doc);
-
-    /* start sending headers away */
-    if (URL->user[0] || URL->pwd[0])
-	_http_basic_auth(f, "Authorization",
-			 URL->user ? URL->user : "",
-			 URL->pwd ? URL->pwd : "");
-    else if ((p = getenv("HTTP_AUTH")) != NULL)
-	_http_authorize(f, "Authorization", p);
-    if (proxy &&  (p = getenv("HTTP_PROXY_AUTH")) != NULL)
-	_http_authorize(f, "Proxy-Authorization", p);
-    _http_cmd(f, "Host: %s:%d" ENDL, host, URL->port);
-    _http_cmd(f, "User-Agent: %s " _LIBFETCH_VER ENDL, __progname);
-    if (URL->offset)
-	_http_cmd(f, "Range: bytes=%lld-" ENDL, URL->offset);
-    _http_cmd(f, "Connection: close" ENDL ENDL);
-
-    /* get response */
-    if ((ln = fgetln(f, &len)) == NULL)
-	return 999;
-    DEBUG(fprintf(stderr, "response: [\033[1m%*.*s\033[m]\n",
-		  (int)len-2, (int)len-2, ln));
-    
-    /* we can't use strchr() and friends since ln isn't NUL-terminated */
-    p = ln;
-    while ((p < ln + len) && !isspace(*p))
-	p++;
-    while ((p < ln + len) && !isdigit(*p))
-	p++;
-    if (!isdigit(*p))
-	return 999;
-    
-    e = atoi(p);
-    DEBUG(fprintf(stderr, "code:     [\033[1m%d\033[m]\n", e));
-    return e;
-}
 
 /*
  * Retrieve a file by HTTP
@@ -529,111 +938,9 @@ _http_request(FILE *f, char *op, struct url *URL, char *flags, int proxy)
 FILE *
 fetchGetHTTP(struct url *URL, char *flags)
 {
-    int e, enc = ENC_NONE, i, noredirect, proxy;
-    struct cookie *c;
-    char *ln, *p, *q;
-    FILE *f, *cf;
-    size_t len;
-    off_t pos = 0;
-
-    noredirect = (flags && strchr(flags, 'A'));
+    struct url_stat us;
     
-    /* allocate cookie */
-    if ((c = calloc(1, sizeof *c)) == NULL)
-	return NULL;
-
-    /* connect */
-    if ((f = _http_connect(URL, flags, &proxy)) == NULL) {
-	free(c);
-	return NULL;
-    }
-    c->real_f = f;
-
-    e = _http_request(f, "GET", URL, flags, proxy);
-    if (e != (URL->offset ? HTTP_PARTIAL : HTTP_OK)
-	&& (e != HTTP_MOVED || noredirect)) {
-	_http_seterr(e);
-	free(c);
-	fclose(f);
-	return NULL;
-    }
-
-    /* browse through header */
-    while (1) {
-	if ((ln = fgetln(f, &len)) == NULL)
-	    goto fouch;
-	if ((ln[0] == '\r') || (ln[0] == '\n'))
-	    break;
-	while (isspace(ln[len-1]))
-	    --len;
-	ln[len] = '\0'; /* XXX */
-	DEBUG(fprintf(stderr, "header:	 [\033[1m%s\033[m]\n", ln));
-	if ((p = _http_match("Location", ln)) != NULL) {
-	    struct url *url;
-	    
-	    for (q = p; *q && !isspace(*q); q++)
-		/* VOID */ ;
-	    *q = 0;
-	    if ((url = fetchParseURL(p)) == NULL)
-		goto fouch;
-	    url->offset = URL->offset;
-	    url->length = URL->length;
-	    DEBUG(fprintf(stderr, "location:  [\033[1m%s\033[m]\n", p));
-	    cf = fetchGetHTTP(url, flags);
-	    fetchFreeURL(url);
-	    fclose(f);
-	    return cf;
-	} else if ((p = _http_match("Transfer-Encoding", ln)) != NULL) {
-	    for (q = p; *q && !isspace(*q); q++)
-		/* VOID */ ;
-	    *q = 0;
-	    if (strcasecmp(p, "chunked") == 0)
-		enc = ENC_CHUNKED;
-	    DEBUG(fprintf(stderr, "transfer encoding:  [\033[1m%s\033[m]\n", p));
-	} else if ((p = _http_match("Content-Type", ln)) != NULL) {
-	    for (i = 0; *p && i < HTTPCTYPELEN; p++, i++)
-		    c->content_type[i] = *p;
-	    do c->content_type[i--] = 0; while (isspace(c->content_type[i]));
-	    DEBUG(fprintf(stderr, "content type: [\033[1m%s\033[m]\n",
-			  c->content_type));
-	} else if ((p = _http_match("Content-Range", ln)) != NULL) {
-	    if (strncasecmp(p, "bytes ", 6) != 0)
-		goto fouch;
-	    p += 6;
-	    while (*p && isdigit(*p))
-		pos = pos * 10 + (*p++ - '0');
-	    /* XXX wouldn't hurt to be slightly more paranoid here */
-	    DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", pos));
-	    if (pos > URL->offset)
-		goto fouch;
-	}
-    }
-
-    /* only body remains */
-    c->encoding = enc;
-    cf = funopen(c,
-		 (int (*)(void *, char *, int))_http_readfn,
-		 (int (*)(void *, const char *, int))_http_writefn,
-		 (fpos_t (*)(void *, fpos_t, int))NULL,
-		 (int (*)(void *))_http_closefn);
-    if (cf == NULL)
-	goto fouch;
-
-    while (pos < URL->offset)
-	if (fgetc(cf) == EOF)
-	    goto cfouch;
-		
-    return cf;
-    
-fouch:
-    fclose(f);
-    free(c);
-    _http_seterr(999); /* XXX do this properly RSN */
-    return NULL;
-cfouch:
-    fclose(cf);
-    _http_seterr(999); /* XXX do this properly RSN */
-    return NULL;
+    return _http_request(URL, "GET", &us, flags);
 }
 
 FILE *
@@ -649,80 +956,12 @@ fetchPutHTTP(struct url *URL, char *flags)
 int
 fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags)
 {
-    int e, noredirect, proxy;
-    size_t len;
-    char *ln, *p, *q;
     FILE *f;
-
-    noredirect = (flags && strchr(flags, 'A'));
-    
-    us->size = -1;
-    us->atime = us->mtime = 0;
     
-    /* connect */
-    if ((f = _http_connect(URL, flags, &proxy)) == NULL)
-	return -1;
-
-    e = _http_request(f, "HEAD", URL, flags, proxy);
-    if (e != HTTP_OK && (e != HTTP_MOVED || noredirect)) {
-	_http_seterr(e);
-	fclose(f);
+    if ((f = _http_request(URL, "HEAD", us, flags)) == NULL)
 	return -1;
-    }
-
-    while (1) {
-	if ((ln = fgetln(f, &len)) == NULL)
-	    goto fouch;
-	if ((ln[0] == '\r') || (ln[0] == '\n'))
-	    break;
-	while (isspace(ln[len-1]))
-	    --len;
-	ln[len] = '\0'; /* XXX */
-	DEBUG(fprintf(stderr, "header:	 [\033[1m%s\033[m]\n", ln));
-	if ((p = _http_match("Location", ln)) != NULL) {
-	    struct url *url;
-	    
-	    for (q = p; *q && !isspace(*q); q++)
-		/* VOID */ ;
-	    *q = 0;
-	    if ((url = fetchParseURL(p)) == NULL)
-		goto ouch;
-	    url->offset = URL->offset;
-	    url->length = URL->length;
-	    DEBUG(fprintf(stderr, "location:  [\033[1m%s\033[m]\n", p));
-	    e = fetchStatHTTP(url, us, flags);
-	    fetchFreeURL(url);
-	    fclose(f);
-	    return e;
-	} else if ((p = _http_match("Last-Modified", ln)) != NULL) {
-	    struct tm tm;
-	    char locale[64];
-
-	    strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale);
-	    setlocale(LC_TIME, "C");
-	    strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
-	    /* XXX should add support for date-2 and date-3 */
-	    setlocale(LC_TIME, locale);
-	    us->atime = us->mtime = timegm(&tm);
-	    DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d "
-			  "%02d:%02d:%02d\033[m]\n",
-			  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
-			  tm.tm_hour, tm.tm_min, tm.tm_sec));
-	} else if ((p = _http_match("Content-Length", ln)) != NULL) {
-	    us->size = 0;
-	    while (*p && isdigit(*p))
-		us->size = us->size * 10 + (*p++ - '0');
-	    DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", us->size));
-	}
-    }
-
     fclose(f);
     return 0;
- ouch:
-    _http_seterr(999); /* XXX do this properly RSN */
- fouch:
-    fclose(f);
-    return -1;    
 }
 
 /*