Merge head from 7/28

6 files changed, 77 insertions, 26 deletions

diff --git a/lib/libfetch/Makefile.depend b/lib/libfetch/Makefile.depend
index 3fcefec..4bedb02 100644
--- a/lib/libfetch/Makefile.depend
+++ b/lib/libfetch/Makefile.depend
@@ -3,6 +3,7 @@
 DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
 
 DIRDEPS = \
+	bin/cat.host \
 	gnu/lib/csu \
 	gnu/lib/libgcc \
 	include \
diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index 5cc7e26..a6fc47c 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -688,6 +688,8 @@ fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
 /*
  * Configure peer verification based on environment.
  */
+#define LOCAL_CERT_FILE	"/usr/local/etc/ssl/cert.pem"
+#define BASE_CERT_FILE	"/etc/ssl/cert.pem"
 static int
 fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
 {
@@ -696,8 +698,12 @@ fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
 	const char *ca_cert_file, *ca_cert_path, *crl_file;
 
 	if (getenv("SSL_NO_VERIFY_PEER") == NULL) {
-		ca_cert_file = getenv("SSL_CA_CERT_FILE") != NULL ?
-		    getenv("SSL_CA_CERT_FILE") : "/etc/ssl/cert.pem";
+		ca_cert_file = getenv("SSL_CA_CERT_FILE");
+		if (ca_cert_file == NULL &&
+		    access(LOCAL_CERT_FILE, R_OK) == 0)
+			ca_cert_file = LOCAL_CERT_FILE;
+		if (ca_cert_file == NULL)
+			ca_cert_file = BASE_CERT_FILE;
 		ca_cert_path = getenv("SSL_CA_CERT_PATH");
 		if (verbose) {
 			fetch_info("Peer verification enabled");
@@ -1104,6 +1110,9 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
 			errno = 0;
 			pfd.revents = 0;
 			if (poll(&pfd, 1, deltams) < 0) {
+				/* POSIX compliance */
+				if (errno == EAGAIN)
+					continue;
 				if (errno == EINTR && fetchRestartCalls)
 					continue;
 				return (-1);
diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h
index 875eef1..41c10d2 100644
--- a/lib/libfetch/common.h
+++ b/lib/libfetch/common.h
@@ -117,6 +117,9 @@ int		 fetch_no_proxy_match(const char *);
  */
 FILE		*http_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
+FILE		*http_request_body(struct url *, const char *,
+		     struct url_stat *, struct url *, const char *,
+		     const char *, const char *);
 FILE		*ftp_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
 
diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index 6b2e4c0..57ecea5 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -627,6 +627,7 @@ the document URL will be used as referrer URL.
 Specifies the User-Agent string to use for HTTP requests.
 This can be useful when working with HTTP origin or proxy servers that
 differentiate between user agents.
+If defined but empty, no User-Agent header is sent.
 .It Ev NETRC
 Specifies a file to use instead of
 .Pa ~/.netrc
@@ -773,27 +774,27 @@ library first appeared in
 The
 .Nm fetch
 library was mostly written by
-.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org
+.An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
 with numerous suggestions and contributions from
-.An Jordan K. Hubbard Aq jkh@FreeBSD.org ,
-.An Eugene Skepner Aq eu@qub.com ,
-.An Hajimu Umemoto Aq ume@FreeBSD.org ,
-.An Henry Whincup Aq henry@techiebod.com ,
-.An Jukka A. Ukkonen Aq jau@iki.fi ,
-.An Jean-Fran\(,cois Dockes Aq jf@dockes.org ,
-.An Michael Gmelin Aq freebsd@grem.de
+.An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org ,
+.An Eugene Skepner Aq Mt eu@qub.com ,
+.An Hajimu Umemoto Aq Mt ume@FreeBSD.org ,
+.An Henry Whincup Aq Mt henry@techiebod.com ,
+.An Jukka A. Ukkonen Aq Mt jau@iki.fi ,
+.An Jean-Fran\(,cois Dockes Aq Mt jf@dockes.org ,
+.An Michael Gmelin Aq Mt freebsd@grem.de
 and others.
 It replaces the older
 .Nm ftpio
 library written by
-.An Poul-Henning Kamp Aq phk@FreeBSD.org
+.An Poul-Henning Kamp Aq Mt phk@FreeBSD.org
 and
-.An Jordan K. Hubbard Aq jkh@FreeBSD.org .
+.An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org .
 .Pp
 This manual page was written by
-.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org
+.An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
 and
-.An Michael Gmelin Aq freebsd@grem.de .
+.An Michael Gmelin Aq Mt freebsd@grem.de .
 .Sh BUGS
 Some parts of the library are not yet implemented.
 The most notable
diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h
index be49482..d56a103 100644
--- a/lib/libfetch/fetch.h
+++ b/lib/libfetch/fetch.h
@@ -102,6 +102,8 @@ FILE		*fetchGetHTTP(struct url *, const char *);
 FILE		*fetchPutHTTP(struct url *, const char *);
 int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListHTTP(struct url *, const char *);
+FILE		*fetchReqHTTP(struct url *, const char *, const char *,
+		    const char *, const char *);
 
 /* FTP-specific functions */
 FILE		*fetchXGetFTP(struct url *, struct url_stat *, const char *);
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index cbbb8a8..ad4a419 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1030,7 +1030,7 @@ typedef struct {
 static void
 init_http_auth_params(http_auth_params_t *s)
 {
-	s->scheme = s->realm = s->user = s->password = 0;
+	s->scheme = s->realm = s->user = s->password = NULL;
 }
 
 static void
@@ -1129,7 +1129,7 @@ CvtHex(IN HASH Bin, OUT HASHHEX Hex)
 		Hex[i*2] = hexchars[j];
 		j = Bin[i] & 0xf;
 		Hex[i*2+1] = hexchars[j];
-	};
+	}
 	Hex[HASHHEXLEN] = '\0';
 };
 
@@ -1164,7 +1164,7 @@ DigestCalcHA1(
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
 		MD5Final(HA1, &Md5Ctx);
-	};
+	}
 	CvtHex(HA1, SessionKey);
 }
 
@@ -1198,7 +1198,7 @@ DigestCalcResponse(
 	if (strcasecmp(pszQop, "auth-int") == 0) {
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, HEntity, HASHHEXLEN);
-	};
+	}
 	MD5Final(HA2, &Md5Ctx);
 	CvtHex(HA2, HA2Hex);
 
@@ -1215,7 +1215,7 @@ DigestCalcResponse(
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszQop, strlen(pszQop));
 		MD5Update(&Md5Ctx, ":", 1);
-	};
+	}
 	MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN);
 	MD5Final(RespHash, &Md5Ctx);
 	CvtHex(RespHash, Response);
@@ -1249,7 +1249,7 @@ http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
 	int r;
 	char noncecount[10];
 	char cnonce[40];
-	char *options = 0;
+	char *options = NULL;
 
 	if (!c->realm || !c->nonce) {
 		DEBUG(fprintf(stderr, "realm/nonce not set in challenge\n"));
@@ -1494,6 +1494,14 @@ http_print_html(FILE *out, FILE *in)
  * Core
  */
 
+FILE *
+http_request(struct url *URL, const char *op, struct url_stat *us,
+	struct url *purl, const char *flags)
+{
+
+	return (http_request_body(URL, op, us, purl, flags, NULL, NULL));
+}
+
 /*
  * Send a request and process the reply
  *
@@ -1501,8 +1509,9 @@ http_print_html(FILE *out, FILE *in)
  * XXX off into a separate function.
  */
 FILE *
-http_request(struct url *URL, const char *op, struct url_stat *us,
-	struct url *purl, const char *flags)
+http_request_body(struct url *URL, const char *op, struct url_stat *us,
+	struct url *purl, const char *flags, const char *content_type,
+	const char *body)
 {
 	char timebuf[80];
 	char hbuf[MAXHOSTNAMELEN + 7], *host;
@@ -1519,6 +1528,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 	http_headerbuf_t headerbuf;
 	http_auth_challenges_t server_challenges;
 	http_auth_challenges_t proxy_challenges;
+	size_t body_len;
 
 	/* The following calls don't allocate anything */
 	init_http_headerbuf(&headerbuf);
@@ -1683,15 +1693,31 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			else
 				http_cmd(conn, "Referer: %s", p);
 		}
-		if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')
-			http_cmd(conn, "User-Agent: %s", p);
-		else
-			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER, getprogname());
+		if ((p = getenv("HTTP_USER_AGENT")) != NULL) {
+			/* no User-Agent if defined but empty */
+			if  (*p != '\0')
+				http_cmd(conn, "User-Agent: %s", p);
+		} else {
+			/* default User-Agent */
+			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
+			    getprogname());
+		}
 		if (url->offset > 0)
 			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
 		http_cmd(conn, "Connection: close");
+
+		if (body) {
+			body_len = strlen(body);
+			http_cmd(conn, "Content-Length: %zu", body_len);
+			if (content_type != NULL)
+				http_cmd(conn, "Content-Type: %s", content_type);
+		}
+
 		http_cmd(conn, "");
 
+		if (body)
+			fetch_write(conn, body, body_len);
+
 		/*
 		 * Force the queued request to be dispatched.  Normally, one
 		 * would do this with shutdown(2) but squid proxies can be
@@ -2042,3 +2068,12 @@ fetchListHTTP(struct url *url __unused, const char *flags __unused)
 	warnx("fetchListHTTP(): not implemented");
 	return (NULL);
 }
+
+FILE *
+fetchReqHTTP(struct url *URL, const char *method, const char *flags,
+	const char *content_type, const char *body)
+{
+
+	return (http_request_body(URL, method, NULL, http_get_proxy(URL, flags),
+	    flags, content_type, body));
+}