MFH (r280630): remove all traces of SSLv2 support

MFH (r285141): remove unused variable
MFH (r288217): correctly check return value from getaddrinfo(3)
MFH (r289419): fix bugs in HTTPS tunnelling
MFH (r289420): use fopen()'s "e" mode instead of fcntl for close-on-exec
MFH (r291453, r291461): use .netrc for http servers and proxies
MFH (r292330, r292332): reset bufpos to 0 after refilling in chunked mode

PR:		194483 199801 193740 204771

4 files changed, 56 insertions, 30 deletions

diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index eabea2b..ae8b79d 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -495,7 +495,8 @@ fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len)
 	hints.ai_protocol = 0;
 	hints.ai_flags = AI_NUMERICHOST;
 	/* port is not relevant for this purpose */
-	getaddrinfo(host, "443", &hints, &res);
+	if (getaddrinfo(host, "443", &hints, &res) != 0)
+		res = NULL;
 	free(host);
 	return res;
 }
@@ -672,9 +673,7 @@ fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
 {
 	long ssl_ctx_options;
 
-	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_TICKET;
-	if (getenv("SSL_ALLOW_SSL2") == NULL)
-		ssl_ctx_options |= SSL_OP_NO_SSLv2;
+	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_TICKET;
 	if (getenv("SSL_ALLOW_SSL3") == NULL)
 		ssl_ctx_options |= SSL_OP_NO_SSLv3;
 	if (getenv("SSL_NO_TLS1") != NULL)
diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index 139363f..0ddc2c2 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 15, 2014
+.Dd November 29, 2015
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -441,10 +441,8 @@ By default
 allows TLSv1 and newer when negotiating the connecting with the remote
 peer.
 You can change this behavior by setting the
-.Ev SSL_ALLOW_SSL2
-and
 .Ev SSL_ALLOW_SSL3
-environment variables to allow SSLv2 and SSLv3, respectively, and
+environment variable to allow SSLv3 and
 .Ev SSL_NO_TLS1 ,
 .Ev SSL_NO_TLS1_1 and
 .Ev SSL_NO_TLS1_2
@@ -633,11 +631,11 @@ If defined but empty, no User-Agent header is sent.
 .It Ev NETRC
 Specifies a file to use instead of
 .Pa ~/.netrc
-to look up login names and passwords for FTP sites.
+to look up login names and passwords for FTP and HTTP sites as well as
+HTTP proxies.
 See
 .Xr ftp 1
 for a description of the file format.
-This feature is experimental.
 .It Ev NO_PROXY
 Either a single asterisk, which disables the use of proxies
 altogether, or a comma- or whitespace-separated list of hosts for
@@ -646,8 +644,6 @@ which proxies should not be used.
 Same as
 .Ev NO_PROXY ,
 for compatibility.
-.It Ev SSL_ALLOW_SSL2
-Allow SSL version 2 when negotiating the connection (not recommended).
 .It Ev SSL_ALLOW_SSL3
 Allow SSL version 3 when negotiating the connection (not recommended).
 .It Ev SSL_CA_CERT_FILE
diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c
index 8c1d404..7b6462f 100644
--- a/lib/libfetch/file.c
+++ b/lib/libfetch/file.c
@@ -48,7 +48,7 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 	if (us && fetchStatFile(u, us, flags) == -1)
 		return (NULL);
 
-	f = fopen(u->doc, "r");
+	f = fopen(u->doc, "re");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -61,7 +61,6 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 
@@ -77,9 +76,9 @@ fetchPutFile(struct url *u, const char *flags)
 	FILE *f;
 
 	if (CHECK_FLAG('a'))
-		f = fopen(u->doc, "a");
+		f = fopen(u->doc, "ae");
 	else
-		f = fopen(u->doc, "w+");
+		f = fopen(u->doc, "w+e");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -92,7 +91,6 @@ fetchPutFile(struct url *u, const char *flags)
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index e0c2979..206648d 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -130,8 +130,8 @@ struct httpio
 	int		 chunked;	/* chunked mode */
 	char		*buf;		/* chunk buffer */
 	size_t		 bufsize;	/* size of chunk buffer */
-	ssize_t		 buflen;	/* amount of data currently in buffer */
-	int		 bufpos;	/* current read offset in buffer */
+	size_t		 buflen;	/* amount of data currently in buffer */
+	size_t		 bufpos;	/* current read offset in buffer */
 	int		 eof;		/* end-of-file flag */
 	int		 error;		/* error flag */
 	size_t		 chunksize;	/* remaining size of current chunk */
@@ -215,6 +215,7 @@ http_fillbuf(struct httpio *io, size_t len)
 	if (io->eof)
 		return (0);
 
+	/* not chunked: just fetch the requested amount */
 	if (io->chunked == 0) {
 		if (http_growbuf(io, len) == -1)
 			return (-1);
@@ -227,6 +228,7 @@ http_fillbuf(struct httpio *io, size_t len)
 		return (io->buflen);
 	}
 
+	/* chunked, but we ran out: get the next chunk header */
 	if (io->chunksize == 0) {
 		switch (http_new_chunk(io)) {
 		case -1:
@@ -238,6 +240,7 @@ http_fillbuf(struct httpio *io, size_t len)
 		}
 	}
 
+	/* fetch the requested amount, but no more than the current chunk */
 	if (len > io->chunksize)
 		len = io->chunksize;
 	if (http_growbuf(io, len) == -1)
@@ -246,8 +249,9 @@ http_fillbuf(struct httpio *io, size_t len)
 		io->error = errno;
 		return (-1);
 	}
+	io->bufpos = 0;
 	io->buflen = nbytes;
-	io->chunksize -= io->buflen;
+	io->chunksize -= nbytes;
 
 	if (io->chunksize == 0) {
 		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
@@ -255,8 +259,6 @@ http_fillbuf(struct httpio *io, size_t len)
 			return (-1);
 	}
 
-	io->bufpos = 0;
-
 	return (io->buflen);
 }
 
@@ -1330,7 +1332,6 @@ static int
 http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 	       http_auth_params_t *parms, struct url *url)
 {
-	http_auth_challenge_t *basic = NULL;
 	http_auth_challenge_t *digest = NULL;
 	int i;
 
@@ -1340,10 +1341,8 @@ http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 		return (-1);
 	}
 
-	/* Look for a Digest and a Basic challenge */
+	/* Look for a Digest */
 	for (i = 0; i < cs->count; i++) {
-		if (cs->challenges[i]->scheme == HTTPAS_BASIC)
-			basic = cs->challenges[i];
 		if (cs->challenges[i]->scheme == HTTPAS_DIGEST)
 			digest = cs->challenges[i];
 	}
@@ -1379,8 +1378,12 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 {
 	struct url *curl;
 	conn_t *conn;
+	hdr_t h;
+	http_headerbuf_t headerbuf;
+	const char *p;
 	int verbose;
 	int af, val;
+	int serrno;
 
 #ifdef INET6
 	af = AF_UNSPEC;
@@ -1401,6 +1404,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
+	init_http_headerbuf(&headerbuf);
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
 		http_cmd(conn, "CONNECT %s:%d HTTP/1.1",
 		    URL->host, URL->port);
@@ -1408,10 +1412,26 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 		    URL->host, URL->port);
 		http_cmd(conn, "");
 		if (http_get_reply(conn) != HTTP_OK) {
-			fetch_close(conn);
-			return (NULL);
+			http_seterr(conn->err);
+			goto ouch;
+		}
+		/* Read and discard the rest of the proxy response */
+		if (fetch_getln(conn) < 0) {
+			fetch_syserr();
+			goto ouch;
 		}
-		http_get_reply(conn);
+		do {
+			switch ((h = http_next_header(conn, &headerbuf, &p))) {
+			case hdr_syserror:
+				fetch_syserr();
+				goto ouch;
+			case hdr_error:
+				http_seterr(HTTP_PROTOCOL_ERROR);
+				goto ouch;
+			default:
+				/* ignore */ ;
+			}
+		} while (h < hdr_end);
 	}
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
@@ -1419,13 +1439,20 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 		/* grrr */
 		errno = EAUTH;
 		fetch_syserr();
-		return (NULL);
+		goto ouch;
 	}
 
 	val = 1;
 	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
 
+	clean_http_headerbuf(&headerbuf);
 	return (conn);
+ouch:
+	serrno = errno;
+	clean_http_headerbuf(&headerbuf);
+	fetch_close(conn);
+	errno = serrno;
+	return (NULL);
 }
 
 static struct url *
@@ -1633,6 +1660,9 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
 					http_seterr(HTTP_NEED_PROXY_AUTH);
 					goto ouch;
 				}
+			} else if (fetch_netrc_auth(purl) == 0) {
+				aparams.user = strdup(purl->user);
+				aparams.password = strdup(purl->pwd);
 			}
 			http_authorize(conn, "Proxy-Authorization",
 				       &proxy_challenges, &aparams, url);
@@ -1660,6 +1690,9 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
 					http_seterr(HTTP_NEED_AUTH);
 					goto ouch;
 				}
+			} else if (fetch_netrc_auth(url) == 0) {
+				aparams.user = strdup(url->user);
+				aparams.password = strdup(url->pwd);
 			} else if (fetchAuthMethod &&
 				   fetchAuthMethod(url) == 0) {
 				aparams.user = strdup(url->user);