From 71362d24e8858b7474378abc08da1b6719cf5a96 Mon Sep 17 00:00:00 2001 From: bdrewery Date: Tue, 19 Nov 2013 15:35:26 +0000 Subject: Support SNI in libfetch SNI is Server Name Indentification which is a protocol for TLS that indicates the host that is being connected to at the start of the handshake. It allows to use Virtual Hosts on HTTPS. Submitted by: sbz Submitted by: Michael Gmelin [1] PR: kern/183583 [1] Reviewed by: des Approved by: bapt MFC after: 1 week --- lib/libfetch/common.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 15d5a25..b15ab9a 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -829,6 +829,15 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) return (-1); } SSL_set_fd(conn->ssl, conn->sd); + +#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) + if (!SSL_set_tlsext_host_name(conn->ssl, URL->host)) { + fprintf(stderr, + "TLS server name indication extension failed for host %s\n", + URL->host); + return (-1); + } +#endif while ((ret = SSL_connect(conn->ssl)) == -1) { ssl_err = SSL_get_error(conn->ssl, ret); if (ssl_err != SSL_ERROR_WANT_READ && -- cgit v1.1 From 6f5000b05b4a5c0ba97b0b8b4e5a9eebcd15c4e0 Mon Sep 17 00:00:00 2001 From: bdrewery Date: Tue, 19 Nov 2013 16:11:03 +0000 Subject: Fix build with GCC SSL_set_tlsext_host_name(3) internally does not modify the host buffer pased to it. So it is safe to DECONST the struct url* here. Reported by: gjb Approved by: bapt (implicit) MFC after: 1 week X-MFC-With: r258347 --- lib/libfetch/common.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index b15ab9a..49d268c 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -831,7 +831,8 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) SSL_set_fd(conn->ssl, conn->sd); #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) - if (!SSL_set_tlsext_host_name(conn->ssl, URL->host)) { + if (!SSL_set_tlsext_host_name(conn->ssl, + __DECONST(struct url *, URL)->host)) { fprintf(stderr, "TLS server name indication extension failed for host %s\n", URL->host); -- cgit v1.1 From 61d98c4b31b800fc3b510fcf220435f006304829 Mon Sep 17 00:00:00 2001 From: des Date: Mon, 20 Jan 2014 11:13:05 +0000 Subject: Fix format string. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Submitted by: Jörg Sonnenberger MFC after: 1 week --- lib/libfetch/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 49d268c..fbacb02 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -679,7 +679,7 @@ fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose) if (getenv("SSL_NO_TLS1") != NULL) ssl_ctx_options |= SSL_OP_NO_TLSv1; if (verbose) - fetch_info("SSL options: %x", ssl_ctx_options); + fetch_info("SSL options: %lx", ssl_ctx_options); SSL_CTX_set_options(ctx, ssl_ctx_options); } -- cgit v1.1 From 2010baa9c1222ce144a45f1d7485b958e9be88fd Mon Sep 17 00:00:00 2001 From: des Date: Tue, 28 Jan 2014 12:48:17 +0000 Subject: Solve http buffering issues and hangs once and for all (hopefully!) by simply not trying to return exactly what the caller asked for - just return whatever we got and let the caller be the judge of whether it was enough. If an error occurs or the connection times out after we already received some data, return a short read, under the assumption that the next call will fail or time out before we read anything. As it turns out, none of the code that calls fetch_read() assumes an all-or-nothing result anyway, except for a couple of lines where we read the CR LF at the end of a hunk in HTTP hunked encoding, so the changes outside of fetch_read() and http_readfn() are minimal. While there, replace select(2) with poll(2). MFC after: 3 days --- lib/libfetch/common.c | 133 +++++++++++++++----------------------------------- lib/libfetch/common.h | 7 --- lib/libfetch/http.c | 38 +++++++-------- 3 files changed, 57 insertions(+), 121 deletions(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index fbacb02..ab81b58 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -641,7 +642,7 @@ fetch_ssl_verify_hname(X509 *cert, const char *host) struct addrinfo *ip; STACK_OF(GENERAL_NAME) *altnames; X509_NAME *subject; - int ret; + int ret; ret = 0; ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host)); @@ -913,33 +914,6 @@ fetch_ssl_read(SSL *ssl, char *buf, size_t len) } #endif -/* - * Cache some data that was read from a socket but cannot be immediately - * returned because of an interrupted system call. - */ -static int -fetch_cache_data(conn_t *conn, char *src, size_t nbytes) -{ - char *tmp; - - if (conn->cache.size < nbytes) { - tmp = realloc(conn->cache.buf, nbytes); - if (tmp == NULL) { - fetch_syserr(); - return (-1); - } - conn->cache.buf = tmp; - conn->cache.size = nbytes; - } - - memcpy(conn->cache.buf, src, nbytes); - conn->cache.len = nbytes; - conn->cache.pos = 0; - - return (0); -} - - static ssize_t fetch_socket_read(int sd, char *buf, size_t len) { @@ -962,46 +936,31 @@ ssize_t fetch_read(conn_t *conn, char *buf, size_t len) { struct timeval now, timeout, delta; - fd_set readfds; - ssize_t rlen, total; - char *start; + struct pollfd pfd; + ssize_t rlen; + int deltams; if (fetchTimeout > 0) { gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } - total = 0; - start = buf; + deltams = INFTIM; + memset(&pfd, 0, sizeof pfd); + pfd.fd = conn->sd; + pfd.events = POLLIN | POLLERR; - if (conn->cache.len > 0) { - /* - * The last invocation of fetch_read was interrupted by a - * signal after some data had been read from the socket. Copy - * the cached data into the supplied buffer before trying to - * read from the socket again. - */ - total = (conn->cache.len < len) ? conn->cache.len : len; - memcpy(buf, conn->cache.buf, total); - - conn->cache.len -= total; - conn->cache.pos += total; - len -= total; - buf += total; - } - - while (len > 0) { + for (;;) { /* * The socket is non-blocking. Instead of the canonical - * select() -> read(), we do the following: + * poll() -> read(), we do the following: * * 1) call read() or SSL_read(). - * 2) if an error occurred, return -1. - * 3) if we received data but we still expect more, - * update our counters and loop. + * 2) if we received some data, return it. + * 3) if an error occurred, return -1. * 4) if read() or SSL_read() signaled EOF, return. * 5) if we did not receive any data but we're not at EOF, - * call select(). + * call poll(). * * In the SSL case, this is necessary because if we * receive a close notification, we have to call @@ -1017,46 +976,34 @@ fetch_read(conn_t *conn, char *buf, size_t len) else #endif rlen = fetch_socket_read(conn->sd, buf, len); - if (rlen == 0) { + if (rlen > 0) { break; - } else if (rlen > 0) { - len -= rlen; - buf += rlen; - total += rlen; - continue; } else if (rlen == FETCH_READ_ERROR) { if (errno == EINTR) - fetch_cache_data(conn, start, total); + break; return (-1); } - // assert(rlen == FETCH_READ_WAIT); - FD_ZERO(&readfds); - while (!FD_ISSET(conn->sd, &readfds)) { - FD_SET(conn->sd, &readfds); - if (fetchTimeout > 0) { - gettimeofday(&now, NULL); - if (!timercmp(&timeout, &now, >)) { - errno = ETIMEDOUT; - fetch_syserr(); - return (-1); - } - timersub(&timeout, &now, &delta); - } - errno = 0; - if (select(conn->sd + 1, &readfds, NULL, NULL, - fetchTimeout > 0 ? &delta : NULL) < 0) { - if (errno == EINTR) { - if (fetchRestartCalls) - continue; - /* Save anything that was read. */ - fetch_cache_data(conn, start, total); - } + if (fetchTimeout > 0) { + gettimeofday(&now, NULL); + if (!timercmp(&timeout, &now, >)) { + errno = ETIMEDOUT; fetch_syserr(); return (-1); } + timersub(&timeout, &now, &delta); + deltams = delta.tv_sec * 1000 + + delta.tv_usec / 1000;; + } + errno = 0; + pfd.revents = 0; + if (poll(&pfd, 1, deltams) < 0) { + if (errno == EINTR && fetchRestartCalls) + continue; + fetch_syserr(); + return (-1); } } - return (total); + return (rlen); } @@ -1130,20 +1077,21 @@ ssize_t fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) { struct timeval now, timeout, delta; - fd_set writefds; + struct pollfd pfd; ssize_t wlen, total; - int r; + int deltams, r; + memset(&pfd, 0, sizeof pfd); if (fetchTimeout) { - FD_ZERO(&writefds); + pfd.fd = conn->sd; + pfd.events = POLLOUT | POLLERR; gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } total = 0; while (iovcnt > 0) { - while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) { - FD_SET(conn->sd, &writefds); + while (fetchTimeout && pfd.revents == 0) { gettimeofday(&now, NULL); delta.tv_sec = timeout.tv_sec - now.tv_sec; delta.tv_usec = timeout.tv_usec - now.tv_usec; @@ -1156,9 +1104,9 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) fetch_syserr(); return (-1); } + deltams = delta.tv_sec * 1000 + delta.tv_usec / 1000;; errno = 0; - r = select(conn->sd + 1, NULL, &writefds, NULL, &delta); - if (r == -1) { + if ((r = poll(&pfd, 1, deltams)) == -1) { if (errno == EINTR && fetchRestartCalls) continue; return (-1); @@ -1250,7 +1198,6 @@ fetch_close(conn_t *conn) } #endif ret = close(conn->sd); - free(conn->cache.buf); free(conn->buf); free(conn); return (ret); diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h index 1d543a6..ba45534 100644 --- a/lib/libfetch/common.h +++ b/lib/libfetch/common.h @@ -52,13 +52,6 @@ struct fetchconn { size_t bufsize; /* buffer size */ size_t buflen; /* length of buffer contents */ int err; /* last protocol reply code */ - struct { /* data cached after an interrupted - read */ - char *buf; - size_t size; - size_t pos; - size_t len; - } cache; #ifdef WITH_SSL SSL *ssl; /* SSL handle */ SSL_CTX *ssl_ctx; /* SSL context */ diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index 87535f0..fc43a97 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -208,6 +208,7 @@ static int http_fillbuf(struct httpio *io, size_t len) { ssize_t nbytes; + char ch; if (io->error) return (-1); @@ -249,10 +250,8 @@ http_fillbuf(struct httpio *io, size_t len) io->chunksize -= io->buflen; if (io->chunksize == 0) { - char endl[2]; - - if (fetch_read(io->conn, endl, 2) != 2 || - endl[0] != '\r' || endl[1] != '\n') + if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' || + fetch_read(io->conn, &ch, 1) != 1 || ch != '\n') return (-1); } @@ -268,31 +267,28 @@ static int http_readfn(void *v, char *buf, int len) { struct httpio *io = (struct httpio *)v; - int l, pos; + int rlen; if (io->error) return (-1); if (io->eof) return (0); - for (pos = 0; len > 0; pos += l, len -= l) { - /* empty buffer */ - if (!io->buf || io->bufpos == io->buflen) - if (http_fillbuf(io, len) < 1) - break; - l = io->buflen - io->bufpos; - if (len < l) - l = len; - memcpy(buf + pos, io->buf + io->bufpos, l); - io->bufpos += l; + /* empty buffer */ + if (!io->buf || io->bufpos == io->buflen) { + if (http_fillbuf(io, len) < 1) { + if (io->error == EINTR) + io->error = 0; + return (-1); + } } - if (!pos && io->error) { - if (io->error == EINTR) - io->error = 0; - return (-1); - } - return (pos); + rlen = io->buflen - io->bufpos; + if (len < rlen) + rlen = len; + memcpy(buf, io->buf + io->bufpos, rlen); + io->bufpos += rlen; + return (rlen); } /* -- cgit v1.1 From 3dc25be505d107528cf27e8a95bfb0b2f10723a7 Mon Sep 17 00:00:00 2001 From: des Date: Wed, 29 Jan 2014 12:48:19 +0000 Subject: r261230 broke the cases where the amount of data to be read is not known in advance, or where the caller doesn't care and just keeps reading until it hits EOF. In fetch_read(): the socket is non-blocking, so read() will return 0 on EOF, and -1 (errno == EAGAIN) when the connection is still open but there is no data waiting. In the first case, we should immediately return 0. The EINTR case was also broken, although not in a way that matters. In fetch_writev(): use timersub() and timercmp() as in fetch_read(). In http_fillbuf(): set errno to a sensible value when an invalid chunk header is encountered. In http_readfn(): as in fetch_read(), a zero return from down the stack indicates EOF, not an error. Furthermore, when io->error is EINTR, clear it (but no errno) before returning so the caller can retry after dealing with the interrupt. MFC after: 3 days --- lib/libfetch/common.c | 23 ++++++++++------------- lib/libfetch/http.c | 10 ++++++---- 2 files changed, 16 insertions(+), 17 deletions(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index ab81b58..94ebfa2 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -976,13 +976,13 @@ fetch_read(conn_t *conn, char *buf, size_t len) else #endif rlen = fetch_socket_read(conn->sd, buf, len); - if (rlen > 0) { + if (rlen >= 0) { break; } else if (rlen == FETCH_READ_ERROR) { - if (errno == EINTR) - break; + fetch_syserr(); return (-1); } + // assert(rlen == FETCH_READ_WAIT); if (fetchTimeout > 0) { gettimeofday(&now, NULL); if (!timercmp(&timeout, &now, >)) { @@ -1079,7 +1079,7 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) struct timeval now, timeout, delta; struct pollfd pfd; ssize_t wlen, total; - int deltams, r; + int deltams; memset(&pfd, 0, sizeof pfd); if (fetchTimeout) { @@ -1093,20 +1093,17 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) while (iovcnt > 0) { while (fetchTimeout && pfd.revents == 0) { gettimeofday(&now, NULL); - delta.tv_sec = timeout.tv_sec - now.tv_sec; - delta.tv_usec = timeout.tv_usec - now.tv_usec; - if (delta.tv_usec < 0) { - delta.tv_usec += 1000000; - delta.tv_sec--; - } - if (delta.tv_sec < 0) { + if (!timercmp(&timeout, &now, >)) { errno = ETIMEDOUT; fetch_syserr(); return (-1); } - deltams = delta.tv_sec * 1000 + delta.tv_usec / 1000;; + timersub(&timeout, &now, &delta); + deltams = delta.tv_sec * 1000 + + delta.tv_usec / 1000; errno = 0; - if ((r = poll(&pfd, 1, deltams)) == -1) { + pfd.revents = 0; + if (poll(&pfd, 1, deltams) < 0) { if (errno == EINTR && fetchRestartCalls) continue; return (-1); diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index fc43a97..676cbb7 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -204,7 +204,7 @@ http_growbuf(struct httpio *io, size_t len) /* * Fill the input buffer, do chunk decoding on the fly */ -static int +static ssize_t http_fillbuf(struct httpio *io, size_t len) { ssize_t nbytes; @@ -230,7 +230,7 @@ http_fillbuf(struct httpio *io, size_t len) if (io->chunksize == 0) { switch (http_new_chunk(io)) { case -1: - io->error = 1; + io->error = EPROTO; return (-1); case 0: io->eof = 1; @@ -276,10 +276,12 @@ http_readfn(void *v, char *buf, int len) /* empty buffer */ if (!io->buf || io->bufpos == io->buflen) { - if (http_fillbuf(io, len) < 1) { - if (io->error == EINTR) + if ((rlen = http_fillbuf(io, len)) < 0) { + if ((errno = io->error) == EINTR) io->error = 0; return (-1); + } else if (rlen == 0) { + return (0); } } -- cgit v1.1 From 8ca08329cac24ebe4162ebae134a49a1871d6709 Mon Sep 17 00:00:00 2001 From: des Date: Thu, 30 Jan 2014 08:37:23 +0000 Subject: Bump copyright dates --- lib/libfetch/common.c | 2 +- lib/libfetch/common.h | 2 +- lib/libfetch/http.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/libfetch') diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 94ebfa2..5cc7e26 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 1998-2011 Dag-Erling Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smørgrav * Copyright (c) 2013 Michael Gmelin * All rights reserved. * diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h index ba45534..875eef1 100644 --- a/lib/libfetch/common.h +++ b/lib/libfetch/common.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 1998-2011 Dag-Erling Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index 676cbb7..b2feedb 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2000-2013 Dag-Erling Smørgrav + * Copyright (c) 2000-2014 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without -- cgit v1.1 From ef589c91da3d72a1fc035e8c70d4196a7c8f2d3e Mon Sep 17 00:00:00 2001 From: bdrewery Date: Tue, 11 Mar 2014 13:47:11 +0000 Subject: Support Last-Modified behind proxies which return UTC instead of GMT. The standard states that GMT must be used, but that UTC is equivalent. Still parse UTC as otherwise this causes problems for pkg(8). It will refetch the repository every time 'pkg update' or other remote operations are used behind these proxies. RFC2616: "All HTTP date/time stamps MUST be represented in Greenwich Mean Time (GMT), without exception. For the purposes of HTTP, GMT is exactly equal to UTC (Coordinated Universal Time)."" Approved by: bapt (mentor) Reviewed by: des, peter Sponsored by: EMC / Isilon Storage Division MFC after: 1 week --- lib/libfetch/http.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib/libfetch') diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index b2feedb..cbbb8a8 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -876,6 +876,12 @@ http_parse_mtime(const char *p, time_t *mtime) strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale)); setlocale(LC_TIME, "C"); r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); + /* + * Some proxies use UTC in response, but it should still be + * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP. + */ + if (r == NULL) + r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm); /* XXX should add support for date-2 and date-3 */ setlocale(LC_TIME, locale); if (r == NULL) -- cgit v1.1