summaryrefslogtreecommitdiffstats
path: root/lib/libfetch
diff options
context:
space:
mode:
Diffstat (limited to 'lib/libfetch')
-rw-r--r--lib/libfetch/http.c1131
1 files changed, 685 insertions, 446 deletions
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index d8ae744..dfe1314 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 1998 Dag-Erling Coïdan Smørgrav
+ * Copyright (c) 2000 Dag-Erling Coïdan Smørgrav
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -12,7 +12,7 @@
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
- * derived from this software without specific prior written permission
+ * derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
@@ -25,46 +25,15 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * $FreeBSD$
+ * $FreeBSD$
*/
-/*
- * The base64 code in this file is based on code from MIT fetch, which
- * has the following copyright and license:
- *
- *-
- * Copyright 1997 Massachusetts Institute of Technology
- *
- * Permission to use, copy, modify, and distribute this software and
- * its documentation for any purpose and without fee is hereby
- * granted, provided that both the above copyright notice and this
- * permission notice appear in all copies, that both the above
- * copyright notice and this permission notice appear in all
- * supporting documentation, and that the name of M.I.T. not be used
- * in advertising or publicity pertaining to distribution of the
- * software without specific, written prior permission. M.I.T. makes
- * no representations about the suitability of this software for any
- * purpose. It is provided "as is" without express or implied
- * warranty.
- *
- * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS
- * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
- * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE. */
-
#include <sys/param.h>
#include <sys/socket.h>
-#include <err.h>
#include <ctype.h>
+#include <err.h>
+#include <errno.h>
#include <locale.h>
#include <netdb.h>
#include <stdarg.h>
@@ -78,148 +47,402 @@
#include "common.h"
#include "httperr.h"
-extern char *__progname;
+extern char *__progname; /* XXX not portable */
-#define ENDL "\r\n"
+/* Maximum number of redirects to follow */
+#define MAX_REDIRECT 5
-#define HTTP_OK 200
-#define HTTP_PARTIAL 206
-#define HTTP_MOVED 302
+/* Symbolic names for reply codes we care about */
+#define HTTP_OK 200
+#define HTTP_PARTIAL 206
+#define HTTP_MOVED_PERM 301
+#define HTTP_MOVED_TEMP 302
+#define HTTP_SEE_OTHER 303
+#define HTTP_NEED_AUTH 401
+#define HTTP_NEED_PROXY_AUTH 403
+#define HTTP_PROTOCOL_ERROR 999
+
+#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
+ || (xyz) == HTTP_MOVED_TEMP \
+ || (xyz) == HTTP_SEE_OTHER)
+
+
+
+/*****************************************************************************
+ * I/O functions for decoding chunked streams
+ */
struct cookie
{
- FILE *real_f;
-#define ENC_NONE 0
-#define ENC_CHUNKED 1
- int encoding; /* 1 = chunked, 0 = none */
-#define HTTPCTYPELEN 59
- char content_type[HTTPCTYPELEN+1];
- char *buf;
- int b_cur, eof;
- unsigned b_len, chunksize;
+ int fd;
+ char *buf;
+ size_t b_size;
+ size_t b_len;
+ int b_pos;
+ int eof;
+ int error;
+ long chunksize;
+#ifdef DEBUG
+ long total;
+#endif
};
/*
- * Send a formatted line; optionally echo to terminal
+ * Get next chunk header
*/
static int
-_http_cmd(FILE *f, char *fmt, ...)
+_http_new_chunk(struct cookie *c)
{
- va_list ap;
-
- va_start(ap, fmt);
- vfprintf(f, fmt, ap);
- DEBUG(fprintf(stderr, "\033[1m>>> "));
- DEBUG(vfprintf(stderr, fmt, ap));
- DEBUG(fprintf(stderr, "\033[m"));
- va_end(ap);
+ char *p;
+
+ if (_fetch_getln(c->fd, &c->buf, &c->b_size, &c->b_len) == -1)
+ return -1;
+
+ if (c->b_len < 2 || !ishexnumber(*c->buf))
+ return -1;
+
+ for (p = c->buf; !isspace(*p) && *p != ';' && p < c->buf + c->b_len; ++p)
+ if (!ishexnumber(*p))
+ return -1;
+ else if (isdigit(*p))
+ c->chunksize = c->chunksize * 16 + *p - '0';
+ else
+ c->chunksize = c->chunksize * 16 + 10 + tolower(*p) - 'a';
+
+#ifdef DEBUG
+ c->total += c->chunksize;
+ if (c->chunksize == 0)
+ fprintf(stderr, "\033[1m_http_fillbuf(): "
+ "end of last chunk\033[m\n");
+ else
+ fprintf(stderr, "\033[1m_http_fillbuf(): "
+ "new chunk: %ld (%ld)\033[m\n", c->chunksize, c->total);
+#endif
- return 0; /* XXX */
+ return c->chunksize;
}
/*
* Fill the input buffer, do chunk decoding on the fly
*/
-static char *
+static int
_http_fillbuf(struct cookie *c)
{
- char *ln;
- unsigned int len;
-
+ if (c->error)
+ return -1;
if (c->eof)
- return NULL;
-
- if (c->encoding == ENC_NONE) {
- c->buf = fgetln(c->real_f, &(c->b_len));
- c->b_cur = 0;
- } else if (c->encoding == ENC_CHUNKED) {
- if (c->chunksize == 0) {
- ln = fgetln(c->real_f, &len);
- if (len <= 2)
- return NULL;
- DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): new chunk: "
- "%*.*s\033[m\n", (int)len-2, (int)len-2, ln));
- sscanf(ln, "%x", &(c->chunksize));
- if (!c->chunksize) {
- DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): "
- "end of last chunk\033[m\n"));
- c->eof = 1;
- return NULL;
- }
- DEBUG(fprintf(stderr, "\033[1m_http_fillbuf(): "
- "new chunk: %X\033[m\n", c->chunksize));
+ return 0;
+
+ if (c->chunksize == 0) {
+ switch (_http_new_chunk(c)) {
+ case -1:
+ c->error = 1;
+ return -1;
+ case 0:
+ c->eof = 1;
+ return 0;
}
- c->buf = fgetln(c->real_f, &(c->b_len));
- if (c->b_len > c->chunksize)
- c->b_len = c->chunksize;
- c->chunksize -= c->b_len;
- c->b_cur = 0;
}
- else return NULL; /* unknown encoding */
- return c->buf;
+
+ if (c->b_size < c->chunksize) {
+ char *tmp;
+
+ if ((tmp = realloc(c->buf, c->chunksize)) == NULL)
+ return -1;
+ c->buf = tmp;
+ c->b_size = c->chunksize;
+ }
+
+ if ((c->b_len = read(c->fd, c->buf, c->chunksize)) == -1)
+ return -1;
+ c->chunksize -= c->b_len;
+
+ if (c->chunksize == 0) {
+ char endl[2];
+ read(c->fd, endl, 2);
+ }
+
+ c->b_pos = 0;
+
+ return c->b_len;
}
/*
* Read function
*/
static int
-_http_readfn(struct cookie *c, char *buf, int len)
+_http_readfn(void *v, char *buf, int len)
{
- int l, pos = 0;
- while (len) {
+ struct cookie *c = (struct cookie *)v;
+ int l, pos;
+
+ if (c->error)
+ return -1;
+ if (c->eof)
+ return 0;
+
+ for (pos = 0; len > 0; pos += l, len -= l) {
/* empty buffer */
- if (!c->buf || (c->b_cur == c->b_len))
- if (!_http_fillbuf(c))
+ if (!c->buf || c->b_pos == c->b_len)
+ if (_http_fillbuf(c) < 1)
break;
-
- l = c->b_len - c->b_cur;
- if (len < l) l = len;
- memcpy(buf + pos, c->buf + c->b_cur, l);
- c->b_cur += l;
- pos += l;
- len -= l;
+ l = c->b_len - c->b_pos;
+ if (len < l)
+ l = len;
+ bcopy(c->buf + c->b_pos, buf + pos, l);
+ c->b_pos += l;
}
-
- if (ferror(c->real_f))
+
+ if (!pos && c->error)
return -1;
- else return pos;
+ return pos;
}
/*
* Write function
*/
static int
-_http_writefn(struct cookie *c, const char *buf, int len)
+_http_writefn(void *v, const char *buf, int len)
{
- size_t r = fwrite(buf, 1, (size_t)len, c->real_f);
- return r ? r : -1;
+ struct cookie *c = (struct cookie *)v;
+
+ return write(c->fd, buf, len);
}
/*
* Close function
*/
static int
-_http_closefn(struct cookie *c)
+_http_closefn(void *v)
{
- int r = fclose(c->real_f);
+ struct cookie *c = (struct cookie *)v;
+ int r;
+
+ r = close(c->fd);
+ if (c->buf)
+ free(c->buf);
free(c);
- return (r == EOF) ? -1 : 0;
+ return r;
+}
+
+/*
+ * Wrap a file descriptor up
+ */
+static FILE *
+_http_funopen(int fd)
+{
+ struct cookie *c;
+ FILE *f;
+
+ if ((c = calloc(1, sizeof *c)) == NULL) {
+ _fetch_syserr();
+ return NULL;
+ }
+ c->fd = fd;
+ if (!(f = funopen(c, _http_readfn, _http_writefn, NULL, _http_closefn))) {
+ _fetch_syserr();
+ free(c);
+ return NULL;
+ }
+ return f;
+}
+
+
+/*****************************************************************************
+ * Helper functions for talking to the server and parsing its replies
+ */
+
+/* Header types */
+typedef enum {
+ hdr_syserror = -2,
+ hdr_error = -1,
+ hdr_end = 0,
+ hdr_unknown = 1,
+ hdr_content_length,
+ hdr_content_range,
+ hdr_last_modified,
+ hdr_location,
+ hdr_transfer_encoding
+} hdr;
+
+/* Names of interesting headers */
+static struct {
+ hdr num;
+ char *name;
+} hdr_names[] = {
+ { hdr_content_length, "Content-Length" },
+ { hdr_content_range, "Content-Range" },
+ { hdr_last_modified, "Last-Modified" },
+ { hdr_location, "Location" },
+ { hdr_transfer_encoding, "Transfer-Encoding" },
+ { hdr_unknown, NULL },
+};
+
+static char *reply_buf;
+static size_t reply_size;
+static size_t reply_length;
+
+/*
+ * Send a formatted line; optionally echo to terminal
+ */
+static int
+_http_cmd(int fd, char *fmt, ...)
+{
+ va_list ap;
+ size_t len;
+ char *msg;
+ int r;
+
+ va_start(ap, fmt);
+ len = vasprintf(&msg, fmt, ap);
+ va_end(ap);
+
+ if (msg == NULL) {
+ errno = ENOMEM;
+ _fetch_syserr();
+ return -1;
+ }
+
+ r = _fetch_putln(fd, msg, len);
+ free(msg);
+
+ if (r == -1) {
+ _fetch_syserr();
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Get and parse status line
+ */
+static int
+_http_get_reply(int fd)
+{
+ if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1)
+ return -1;
+ /*
+ * A valid status line looks like "HTTP/m.n xyz reason" where m
+ * and n are the major and minor protocol version numbers and xyz
+ * is the reply code.
+ * We grok HTTP 1.0 and 1.1, so m must be 1 and n must be 0 or 1.
+ * We don't care about the reason phrase.
+ */
+ if (strncmp(reply_buf, "HTTP/1.", 7) != 0
+ || (reply_buf[7] != '0' && reply_buf[7] != '1') || reply_buf[8] != ' '
+ || !isdigit(reply_buf[9])
+ || !isdigit(reply_buf[10])
+ || !isdigit(reply_buf[11]))
+ return HTTP_PROTOCOL_ERROR;
+
+ return ((reply_buf[9] - '0') * 100
+ + (reply_buf[10] - '0') * 10
+ + (reply_buf[11] - '0'));
}
/*
- * Extract content type from cookie
+ * Check a header; if the type matches the given string, return a
+ * pointer to the beginning of the value.
*/
-char *
-fetchContentType(FILE *f)
+static char *
+_http_match(char *str, char *hdr)
{
+ while (*str && *hdr && tolower(*str++) == tolower(*hdr++))
+ /* nothing */;
+ if (*str || *hdr != ':')
+ return NULL;
+ while (*hdr && isspace(*++hdr))
+ /* nothing */;
+ return hdr;
+}
+
+/*
+ * Get the next header and return the appropriate symbolic code.
+ */
+static hdr
+_http_next_header(int fd, char **p)
+{
+ int i;
+
+ if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1)
+ return hdr_syserror;
+ while (reply_length && isspace(reply_buf[reply_length-1]))
+ reply_length--;
+ reply_buf[reply_length] = 0;
+ if (reply_length == 0)
+ return hdr_end;
/*
- * We have no way of making sure this really *is* one of our cookies,
- * so just check for a null pointer and hope for the best.
+ * We could check for malformed headers but we don't really care.
+ * A valid header starts with a token immediately followed by a
+ * colon; a token is any sequence of non-control, non-whitespace
+ * characters except "()<>@,;:\\\"{}".
*/
- return f->_cookie ? (((struct cookie *)f->_cookie)->content_type) : NULL;
+ for (i = 0; hdr_names[i].num != hdr_unknown; i++)
+ if ((*p = _http_match(hdr_names[i].name, reply_buf)) != NULL)
+ return hdr_names[i].num;
+ return hdr_unknown;
+}
+
+/*
+ * Parse a last-modified header
+ */
+static time_t
+_http_parse_mtime(char *p)
+{
+ char locale[64];
+ struct tm tm;
+
+ strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale);
+ setlocale(LC_TIME, "C");
+ strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+ /* XXX should add support for date-2 and date-3 */
+ setlocale(LC_TIME, locale);
+ DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d "
+ "%02d:%02d:%02d\033[m]\n",
+ tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+ tm.tm_hour, tm.tm_min, tm.tm_sec));
+ return timegm(&tm);
}
/*
+ * Parse a content-length header
+ */
+static off_t
+_http_parse_length(char *p)
+{
+ off_t len;
+
+ for (len = 0; *p && isdigit(*p); ++p)
+ len = len * 10 + (*p - '0');
+ DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", len));
+ return len;
+}
+
+/*
+ * Parse a content-range header
+ */
+static off_t
+_http_parse_range(char *p)
+{
+ off_t off;
+
+ if (strncasecmp(p, "bytes ", 6) != 0)
+ return -1;
+ for (p += 6, off = 0; *p && isdigit(*p); ++p)
+ off = off * 10 + *p - '0';
+ if (*p != '-')
+ return -1;
+ DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", off));
+ return off;
+}
+
+
+/*****************************************************************************
+ * Helper functions for authorization
+ */
+
+/*
* Base64 encoding
*/
static char *
@@ -279,7 +502,7 @@ _http_base64(char *src)
* Encode username and password
*/
static int
-_http_basic_auth(FILE *f, char *hdr, char *usr, char *pwd)
+_http_basic_auth(int fd, char *hdr, char *usr, char *pwd)
{
char *upw, *auth;
int r;
@@ -290,7 +513,7 @@ _http_basic_auth(FILE *f, char *hdr, char *usr, char *pwd)
free(upw);
if (auth == NULL)
return -1;
- r = _http_cmd(f, "%s: Basic %s" ENDL, hdr, auth);
+ r = _http_cmd(fd, "%s: Basic %s", hdr, auth);
free(auth);
return r;
}
@@ -299,7 +522,7 @@ _http_basic_auth(FILE *f, char *hdr, char *usr, char *pwd)
* Send an authorization header
*/
static int
-_http_authorize(FILE *f, char *hdr, char *p)
+_http_authorize(int fd, char *hdr, char *p)
{
/* basic authorization */
if (strncasecmp(p, "basic:", 6) == 0) {
@@ -316,40 +539,138 @@ _http_authorize(FILE *f, char *hdr, char *p)
user = str;
pwd = strchr(str, ':');
*pwd++ = '\0';
- r = _http_basic_auth(f, hdr, user, pwd);
+ r = _http_basic_auth(fd, hdr, user, pwd);
free(str);
return r;
}
return -1;
}
+
+/*****************************************************************************
+ * Helper functions for connecting to a server or proxy
+ */
+
/*
- * Connect to server or proxy
+ * Connect to the specified HTTP proxy server.
*/
-static FILE *
-_http_connect(struct url *URL, char *flags, int *proxy)
+static int
+_http_proxy_connect(char *proxy, int af, int verbose)
{
- int direct, sd = -1, verbose;
+ char *hostname, *p;
+ int fd, port;
+
+ /* get hostname */
+ hostname = NULL;
#ifdef INET6
- int af = AF_UNSPEC;
-#else
- int af = AF_INET;
+ /* host part can be an IPv6 address enclosed in square brackets */
+ if (*proxy == '[') {
+ if ((p = strchr(proxy, ']')) == NULL) {
+ /* no terminating bracket */
+ /* XXX should set an error code */
+ goto ouch;
+ }
+ if (p[1] != '\0' && p[1] != ':') {
+ /* garbage after address */
+ /* XXX should set an error code */
+ goto ouch;
+ }
+ if ((hostname = malloc(p - proxy)) == NULL) {
+ errno = ENOMEM;
+ _fetch_syserr();
+ goto ouch;
+ }
+ strncpy(hostname, proxy + 1, p - proxy - 1);
+ hostname[p - proxy - 1] = '\0';
+ ++p;
+ } else {
+#endif /* INET6 */
+ if ((p = strchr(proxy, ':')) == NULL)
+ p = strchr(proxy, '\0');
+ if ((hostname = malloc(p - proxy + 1)) == NULL) {
+ errno = ENOMEM;
+ _fetch_syserr();
+ goto ouch;
+ }
+ strncpy(hostname, proxy, p - proxy);
+ hostname[p - proxy] = '\0';
+#ifdef INET6
+ }
+#endif /* INET6 */
+ DEBUG(fprintf(stderr, "proxy name: [%s]\n", hostname));
+
+ /* get port number */
+ port = 0;
+ if (*p == ':') {
+ ++p;
+ if (strspn(p, "0123456789") != strlen(p) || strlen(p) > 5) {
+ /* port number is non-numeric or too long */
+ /* XXX should set an error code */
+ goto ouch;
+ }
+ port = atoi(p);
+ if (port < 1 || port > 65535) {
+ /* port number is out of range */
+ /* XXX should set an error code */
+ goto ouch;
+ }
+ }
+
+ if (!port) {
+#if 0
+ /*
+ * commented out, since there is currently no service name
+ * for HTTP proxies
+ */
+ struct servent *se;
+
+ if ((se = getservbyname("xxxx", "tcp")) != NULL)
+ port = ntohs(se->s_port);
+ else
#endif
- size_t len;
- char *px;
- FILE *f;
+ port = 3128;
+ }
+ DEBUG(fprintf(stderr, "proxy port: %d\n", port));
+
+ /* connect */
+ if ((fd = _fetch_connect(hostname, port, af, verbose)) == -1)
+ _fetch_syserr();
+ return fd;
+ ouch:
+ if (hostname)
+ free(hostname);
+ return -1;
+}
+
+/*
+ * Connect to the correct HTTP server or proxy.
+ */
+static int
+_http_connect(struct url *URL, int *proxy, char *flags)
+{
+ int direct, verbose;
+ int af, fd;
+ char *p;
+
+#ifdef INET6
+ af = AF_UNSPEC;
+#else
+ af = AF_INET;
+#endif
+
direct = (flags && strchr(flags, 'd'));
verbose = (flags && strchr(flags, 'v'));
- if ((flags && strchr(flags, '4')))
+ if (flags && strchr(flags, '4'))
af = AF_INET;
- else if ((flags && strchr(flags, '6')))
+ else if (flags && strchr(flags, '6'))
af = AF_INET6;
/* check port */
if (!URL->port) {
struct servent *se;
+ /* Scheme can be ftp if we're using a proxy */
if (strcasecmp(URL->scheme, "ftp") == 0)
if ((se = getservbyname("ftp", "tcp")) != NULL)
URL->port = ntohs(se->s_port);
@@ -362,166 +683,254 @@ _http_connect(struct url *URL, char *flags, int *proxy)
URL->port = 80;
}
- /* attempt to connect to proxy server */
- if (!direct && (px = getenv("HTTP_PROXY")) != NULL) {
- char host[MAXHOSTNAMELEN];
- int port = 0;
+ if (!direct && (p = getenv("HTTP_PROXY")) != NULL) {
+ /* attempt to connect to proxy server */
+ if ((fd = _http_proxy_connect(p, af, verbose)) == -1)
+ return -1;
+ *proxy = 1;
+ } else {
+ /* if no proxy is configured, try direct */
+ if (strcasecmp(URL->scheme, "ftp") == 0) {
+ /* can't talk http to an ftp server */
+ /* XXX should set an error code */
+ return -1;
+ }
+ if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1)
+ /* _fetch_connect() has already set an error code */
+ return -1;
+ *proxy = 0;
+ }
- /* measure length */
+ return fd;
+}
+
+
+/*****************************************************************************
+ * Core
+ */
+
+/*
+ * Send a request and process the reply
+ */
+static FILE *
+_http_request(struct url *URL, char *op, struct url_stat *us, char *flags)
+{
+ struct url *url, *new;
+ int chunked, need_auth, noredirect, proxy, verbose;
+ int code, fd, i, n;
+ off_t offset;
+ char *p;
+ FILE *f;
+ hdr h;
+ char *host;
#ifdef INET6
- if (px[0] != '[' ||
- (len = strcspn(px, "]")) >= strlen(px) ||
- (px[++len] != '\0' && px[len] != ':'))
+ char hbuf[MAXHOSTNAMELEN + 1];
#endif
- len = strcspn(px, ":");
- /* get port (XXX atoi is a little too tolerant perhaps?) */
- if (px[len] == ':') {
- if (strspn(px+len+1, "0123456789") != strlen(px+len+1)
- || strlen(px+len+1) > 5) {
- /* XXX we should emit some kind of warning */
- }
- port = atoi(px+len+1);
- if (port < 1 || port > 65535) {
- /* XXX we should emit some kind of warning */
+ noredirect = (flags && strchr(flags, 'A'));
+ verbose = (flags && strchr(flags, 'v'));
+
+ n = noredirect ? 1 : MAX_REDIRECT;
+
+ us->size = -1;
+ us->atime = us->mtime = 0;
+ chunked = 0;
+ offset = 0;
+ fd = -1;
+ code = HTTP_PROTOCOL_ERROR; /* just to appease a compiler warning */
+
+ for (url = URL, i = 0; i < n; ++i) {
+ need_auth = 0;
+ retry:
+ /* connect to server or proxy */
+ if ((fd = _http_connect(url, &proxy, flags)) == -1)
+ goto ouch;
+
+ host = url->host;
+#ifdef INET6
+ if (strchr(url->host, ':')) {
+ snprintf(hbuf, sizeof(hbuf), "[%s]", url->host);
+ host = hbuf;
+ }
+#endif
+
+ /* send request */
+ if (verbose)
+ _fetch_info("requesting %s://%s:%d%s",
+ url->scheme, host, url->port, url->doc);
+ if (proxy) {
+ _http_cmd(fd, "%s %s://%s:%d%s HTTP/1.1",
+ op, url->scheme, host, url->port, url->doc);
+ } else {
+ _http_cmd(fd, "%s %s HTTP/1.1",
+ op, url->doc);
+ }
+
+ /* proxy authorization */
+ if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL)
+ _http_authorize(fd, "Proxy-Authorization", p);
+
+ /* server authorization */
+ if (need_auth) {
+ if (*url->user || *url->pwd)
+ _http_basic_auth(fd, "Authorization",
+ url->user ? url->user : "",
+ url->pwd ? url->pwd : "");
+ else if ((p = getenv("HTTP_AUTH")) != NULL)
+ _http_authorize(fd, "Authorization", p);
+ else {
+ _http_seterr(HTTP_NEED_AUTH);
+ goto ouch;
}
}
- if (!port) {
-#if 0
+
+ /* other headers */
+ _http_cmd(fd, "Host: %s:%d", host, url->port);
+ _http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname);
+ if (URL->offset)
+ _http_cmd(fd, "Range: bytes=%lld-", url->offset);
+ _http_cmd(fd, "Connection: close");
+ _http_cmd(fd, "");
+
+ /* get reply */
+ switch ((code = _http_get_reply(fd))) {
+ case HTTP_OK:
+ case HTTP_PARTIAL:
+ /* fine */
+ break;
+ case HTTP_MOVED_PERM:
+ case HTTP_MOVED_TEMP:
/*
- * commented out, since there is currently no service name
- * for HTTP proxies
+ * Not so fine, but we still have to read the headers to
+ * get the new location.
*/
- struct servent *se;
-
- if ((se = getservbyname("xxxx", "tcp")) != NULL)
- port = ntohs(se->s_port);
- else
-#endif
- port = 3128;
+ break;
+ case HTTP_NEED_AUTH:
+ if (need_auth) {
+ /*
+ * We already sent out authorization code, so there's
+ * nothing more we can do.
+ */
+ _http_seterr(code);
+ goto ouch;
+ }
+ /* try again, but send the password this time */
+ if (verbose)
+ _fetch_info("server requires authorization");
+ need_auth = 1;
+ close(fd);
+ goto retry;
+ case HTTP_NEED_PROXY_AUTH:
+ /*
+ * If we're talking to a proxy, we already sent our proxy
+ * authorization code, so there's nothing more we can do.
+ */
+ _http_seterr(code);
+ goto ouch;
+ case HTTP_PROTOCOL_ERROR:
+ /* fall through */
+ case -1:
+ _fetch_syserr();
+ goto ouch;
+ default:
+ _http_seterr(code);
+ goto ouch;
}
- /* get host name */
-#ifdef INET6
- if (len > 1 && px[0] == '[' && px[len - 1] == ']') {
- px++;
- len -= 2;
- }
-#endif
- if (len >= MAXHOSTNAMELEN)
- len = MAXHOSTNAMELEN - 1;
- strncpy(host, px, len);
- host[len] = 0;
+ /* get headers */
+ do {
+ switch ((h = _http_next_header(fd, &p))) {
+ case hdr_syserror:
+ _fetch_syserr();
+ goto ouch;
+ case hdr_error:
+ _http_seterr(HTTP_PROTOCOL_ERROR);
+ goto ouch;
+ case hdr_content_length:
+ us->size = _http_parse_length(p);
+ break;
+ case hdr_content_range:
+ offset = _http_parse_range(p);
+ break;
+ case hdr_last_modified:
+ us->atime = us->mtime = _http_parse_mtime(p);
+ break;
+ case hdr_location:
+ if (!HTTP_REDIRECT(code))
+ break;
+ if (verbose)
+ _fetch_info("%d redirect to %s", code, p);
+ if ((new = fetchParseURL(p)) == NULL)
+ /* invalid location */
+ goto ouch;
+ if (!*new->user && !*new->pwd) {
+ strcpy(new->user, url->user);
+ strcpy(new->pwd, url->pwd);
+ }
+ new->offset = url->offset;
+ new->length = url->length;
+ close(fd);
+ us->size = -1;
+ us->atime = us->mtime = 0;
+ chunked = 0;
+ offset = 0;
+ fd = -1;
+ if (url != URL)
+ fetchFreeURL(url);
+ url = new;
+ continue;
+ case hdr_transfer_encoding:
+ /* XXX weak test*/
+ chunked = (strcasecmp(p, "chunked") == 0);
+ break;
+ case hdr_end:
+ /* fall through */
+ case hdr_unknown:
+ /* ignore */
+ break;
+ }
+ } while (h > hdr_end);
- /* connect */
- sd = _fetch_connect(host, port, af, verbose);
+ if (code == HTTP_OK || code == HTTP_PARTIAL)
+ break;
}
- /* if no proxy is configured or could be contacted, try direct */
- *proxy = (sd != -1);
- if (!*proxy) {
- if (strcasecmp(URL->scheme, "ftp") == 0)
- goto ouch;
- if ((sd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1)
- goto ouch;
+ /* no success */
+ if (fd == -1) {
+ _http_seterr(code);
+ goto ouch;
}
- /* reopen as stream */
- if ((f = fdopen(sd, "r+")) == NULL)
+ /* wrap it up in a FILE */
+ if ((f = chunked ? _http_funopen(fd) : fdopen(fd, "r")) == NULL) {
+ _fetch_syserr();
goto ouch;
+ }
+
+ while (offset++ < url->offset)
+ if (fgetc(f) == EOF) {
+ _fetch_syserr();
+ fclose(f);
+ f = NULL;
+ }
+
+ if (url != URL)
+ fetchFreeURL(url);
return f;
-ouch:
- if (sd >= 0)
- close(sd);
- _http_seterr(999); /* XXX do this properly RSN */
+ ouch:
+ if (url != URL)
+ fetchFreeURL(url);
+ if (fd != -1)
+ close(fd);
return NULL;
}
-/*
- * Check a header line
- */
-static char *
-_http_match(char *str, char *hdr)
-{
- while (*str && *hdr && tolower(*str++) == tolower(*hdr++))
- /* nothing */;
- if (*str || *hdr != ':')
- return NULL;
- while (*hdr && isspace(*++hdr))
- /* nothing */;
- return hdr;
-}
-
-/*
- * Send a HEAD or GET request
+
+/*****************************************************************************
+ * Entry points
*/
-static int
-_http_request(FILE *f, char *op, struct url *URL, char *flags, int proxy)
-{
- int e, verbose;
- char *ln, *p;
- size_t len;
- char *host;
-#ifdef INET6
- char hbuf[MAXHOSTNAMELEN + 1];
-#endif
-
- verbose = (flags && strchr(flags, 'v'));
-
- host = URL->host;
-#ifdef INET6
- if (strchr(URL->host, ':')) {
- snprintf(hbuf, sizeof(hbuf), "[%s]", URL->host);
- host = hbuf;
- }
-#endif
-
- /* send request (proxies require absolute form) */
- if (verbose)
- _fetch_info("requesting %s://%s:%d%s",
- URL->scheme, host, URL->port, URL->doc);
- if (proxy)
- _http_cmd(f, "%s %s://%s:%d%s HTTP/1.1" ENDL,
- op, URL->scheme, host, URL->port, URL->doc);
- else
- _http_cmd(f, "%s %s HTTP/1.1" ENDL, op, URL->doc);
-
- /* start sending headers away */
- if (URL->user[0] || URL->pwd[0])
- _http_basic_auth(f, "Authorization",
- URL->user ? URL->user : "",
- URL->pwd ? URL->pwd : "");
- else if ((p = getenv("HTTP_AUTH")) != NULL)
- _http_authorize(f, "Authorization", p);
- if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL)
- _http_authorize(f, "Proxy-Authorization", p);
- _http_cmd(f, "Host: %s:%d" ENDL, host, URL->port);
- _http_cmd(f, "User-Agent: %s " _LIBFETCH_VER ENDL, __progname);
- if (URL->offset)
- _http_cmd(f, "Range: bytes=%lld-" ENDL, URL->offset);
- _http_cmd(f, "Connection: close" ENDL ENDL);
-
- /* get response */
- if ((ln = fgetln(f, &len)) == NULL)
- return 999;
- DEBUG(fprintf(stderr, "response: [\033[1m%*.*s\033[m]\n",
- (int)len-2, (int)len-2, ln));
-
- /* we can't use strchr() and friends since ln isn't NUL-terminated */
- p = ln;
- while ((p < ln + len) && !isspace(*p))
- p++;
- while ((p < ln + len) && !isdigit(*p))
- p++;
- if (!isdigit(*p))
- return 999;
-
- e = atoi(p);
- DEBUG(fprintf(stderr, "code: [\033[1m%d\033[m]\n", e));
- return e;
-}
/*
* Retrieve a file by HTTP
@@ -529,111 +938,9 @@ _http_request(FILE *f, char *op, struct url *URL, char *flags, int proxy)
FILE *
fetchGetHTTP(struct url *URL, char *flags)
{
- int e, enc = ENC_NONE, i, noredirect, proxy;
- struct cookie *c;
- char *ln, *p, *q;
- FILE *f, *cf;
- size_t len;
- off_t pos = 0;
-
- noredirect = (flags && strchr(flags, 'A'));
+ struct url_stat us;
- /* allocate cookie */
- if ((c = calloc(1, sizeof *c)) == NULL)
- return NULL;
-
- /* connect */
- if ((f = _http_connect(URL, flags, &proxy)) == NULL) {
- free(c);
- return NULL;
- }
- c->real_f = f;
-
- e = _http_request(f, "GET", URL, flags, proxy);
- if (e != (URL->offset ? HTTP_PARTIAL : HTTP_OK)
- && (e != HTTP_MOVED || noredirect)) {
- _http_seterr(e);
- free(c);
- fclose(f);
- return NULL;
- }
-
- /* browse through header */
- while (1) {
- if ((ln = fgetln(f, &len)) == NULL)
- goto fouch;
- if ((ln[0] == '\r') || (ln[0] == '\n'))
- break;
- while (isspace(ln[len-1]))
- --len;
- ln[len] = '\0'; /* XXX */
- DEBUG(fprintf(stderr, "header: [\033[1m%s\033[m]\n", ln));
- if ((p = _http_match("Location", ln)) != NULL) {
- struct url *url;
-
- for (q = p; *q && !isspace(*q); q++)
- /* VOID */ ;
- *q = 0;
- if ((url = fetchParseURL(p)) == NULL)
- goto fouch;
- url->offset = URL->offset;
- url->length = URL->length;
- DEBUG(fprintf(stderr, "location: [\033[1m%s\033[m]\n", p));
- cf = fetchGetHTTP(url, flags);
- fetchFreeURL(url);
- fclose(f);
- return cf;
- } else if ((p = _http_match("Transfer-Encoding", ln)) != NULL) {
- for (q = p; *q && !isspace(*q); q++)
- /* VOID */ ;
- *q = 0;
- if (strcasecmp(p, "chunked") == 0)
- enc = ENC_CHUNKED;
- DEBUG(fprintf(stderr, "transfer encoding: [\033[1m%s\033[m]\n", p));
- } else if ((p = _http_match("Content-Type", ln)) != NULL) {
- for (i = 0; *p && i < HTTPCTYPELEN; p++, i++)
- c->content_type[i] = *p;
- do c->content_type[i--] = 0; while (isspace(c->content_type[i]));
- DEBUG(fprintf(stderr, "content type: [\033[1m%s\033[m]\n",
- c->content_type));
- } else if ((p = _http_match("Content-Range", ln)) != NULL) {
- if (strncasecmp(p, "bytes ", 6) != 0)
- goto fouch;
- p += 6;
- while (*p && isdigit(*p))
- pos = pos * 10 + (*p++ - '0');
- /* XXX wouldn't hurt to be slightly more paranoid here */
- DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", pos));
- if (pos > URL->offset)
- goto fouch;
- }
- }
-
- /* only body remains */
- c->encoding = enc;
- cf = funopen(c,
- (int (*)(void *, char *, int))_http_readfn,
- (int (*)(void *, const char *, int))_http_writefn,
- (fpos_t (*)(void *, fpos_t, int))NULL,
- (int (*)(void *))_http_closefn);
- if (cf == NULL)
- goto fouch;
-
- while (pos < URL->offset)
- if (fgetc(cf) == EOF)
- goto cfouch;
-
- return cf;
-
-fouch:
- fclose(f);
- free(c);
- _http_seterr(999); /* XXX do this properly RSN */
- return NULL;
-cfouch:
- fclose(cf);
- _http_seterr(999); /* XXX do this properly RSN */
- return NULL;
+ return _http_request(URL, "GET", &us, flags);
}
FILE *
@@ -649,80 +956,12 @@ fetchPutHTTP(struct url *URL, char *flags)
int
fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags)
{
- int e, noredirect, proxy;
- size_t len;
- char *ln, *p, *q;
FILE *f;
-
- noredirect = (flags && strchr(flags, 'A'));
-
- us->size = -1;
- us->atime = us->mtime = 0;
- /* connect */
- if ((f = _http_connect(URL, flags, &proxy)) == NULL)
- return -1;
-
- e = _http_request(f, "HEAD", URL, flags, proxy);
- if (e != HTTP_OK && (e != HTTP_MOVED || noredirect)) {
- _http_seterr(e);
- fclose(f);
+ if ((f = _http_request(URL, "HEAD", us, flags)) == NULL)
return -1;
- }
-
- while (1) {
- if ((ln = fgetln(f, &len)) == NULL)
- goto fouch;
- if ((ln[0] == '\r') || (ln[0] == '\n'))
- break;
- while (isspace(ln[len-1]))
- --len;
- ln[len] = '\0'; /* XXX */
- DEBUG(fprintf(stderr, "header: [\033[1m%s\033[m]\n", ln));
- if ((p = _http_match("Location", ln)) != NULL) {
- struct url *url;
-
- for (q = p; *q && !isspace(*q); q++)
- /* VOID */ ;
- *q = 0;
- if ((url = fetchParseURL(p)) == NULL)
- goto ouch;
- url->offset = URL->offset;
- url->length = URL->length;
- DEBUG(fprintf(stderr, "location: [\033[1m%s\033[m]\n", p));
- e = fetchStatHTTP(url, us, flags);
- fetchFreeURL(url);
- fclose(f);
- return e;
- } else if ((p = _http_match("Last-Modified", ln)) != NULL) {
- struct tm tm;
- char locale[64];
-
- strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale);
- setlocale(LC_TIME, "C");
- strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
- /* XXX should add support for date-2 and date-3 */
- setlocale(LC_TIME, locale);
- us->atime = us->mtime = timegm(&tm);
- DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d "
- "%02d:%02d:%02d\033[m]\n",
- tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
- tm.tm_hour, tm.tm_min, tm.tm_sec));
- } else if ((p = _http_match("Content-Length", ln)) != NULL) {
- us->size = 0;
- while (*p && isdigit(*p))
- us->size = us->size * 10 + (*p++ - '0');
- DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", us->size));
- }
- }
-
fclose(f);
return 0;
- ouch:
- _http_seterr(999); /* XXX do this properly RSN */
- fouch:
- fclose(f);
- return -1;
}
/*
OpenPOWER on IntegriCloud