Implement certificate verification, and many other SSL-related

imrovements; complete details in the PR.

PR:		kern/175514
Submitted by:	Michael Gmelin <freebsd@grem.de>
MFC after:	1 week

2 files changed, 273 insertions, 35 deletions

diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1
index cea6f1b..c340649 100644
--- a/usr.bin/fetch/fetch.1
+++ b/usr.bin/fetch/fetch.1
@@ -38,22 +38,51 @@
 .Sh SYNOPSIS
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
+.Op Fl -allow-sslv2
 .Op Fl B Ar bytes
+.Op Fl -bind-address= Ns Ar host
+.Op Fl -ca-cert= Ns Ar file
+.Op Fl -ca-path= Ns Ar dir
+.Op Fl -cert= Ns Ar file
+.Op Fl -crl= Ns Ar file
 .Op Fl i Ar file
+.Op Fl -key= Ns Ar file
 .Op Fl N Ar file
+.Op Fl -no-passive
+.Op Fl -no-proxy= Ns Ar list
+.Op Fl -no-sslv3
+.Op Fl -no-tlsv1
+.Op Fl -no-verify-hostname
+.Op Fl -no-verify-peer
 .Op Fl o Ar file
+.Op Fl -referer= Ns Ar URL
 .Op Fl S Ar bytes
 .Op Fl T Ar seconds
+.Op Fl -user-agent= Ns Ar agent-string
 .Op Fl w Ar seconds
 .Ar URL ...
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
 .Op Fl B Ar bytes
+.Op Fl -bind-address= Ns Ar host
+.Op Fl -ca-cert= Ns Ar file
+.Op Fl -ca-path= Ns Ar dir
+.Op Fl -cert= Ns Ar file
+.Op Fl -crl= Ns Ar file
 .Op Fl i Ar file
+.Op Fl -key= Ns Ar file
 .Op Fl N Ar file
+.Op Fl -no-passive
+.Op Fl -no-proxy= Ns Ar list
+.Op Fl -no-sslv3
+.Op Fl -no-tlsv1
+.Op Fl -no-verify-hostname
+.Op Fl -no-verify-peer
 .Op Fl o Ar file
+.Op Fl -referer= Ns Ar URL
 .Op Fl S Ar bytes
 .Op Fl T Ar seconds
+.Op Fl -user-agent= Ns Ar agent-string
 .Op Fl w Ar seconds
 .Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc
 .Sh DESCRIPTION
@@ -67,23 +96,26 @@ command line.
 .Pp
 The following options are available:
 .Bl -tag -width Fl
-.It Fl 1
+.It Fl 1 , -one-file
 Stop and return exit code 0 at the first successfully retrieved file.
-.It Fl 4
+.It Fl 4 , -ipv4-only
 Forces
 .Nm
 to use IPv4 addresses only.
-.It Fl 6
+.It Fl 6 , -ipv6-only
 Forces
 .Nm
 to use IPv6 addresses only.
-.It Fl A
+.It Fl A , -no-redirect
 Do not automatically follow ``temporary'' (302) redirects.
 Some broken Web sites will return a redirect instead of a not-found
 error when the requested object does not exist.
-.It Fl a
+.It Fl a , -retry
 Automatically retry the transfer upon soft failures.
-.It Fl B Ar bytes
+.It Fl -allow-sslv2
+[SSL]
+Allow SSL version 2 when negotiating the connection.
+.It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes
 Specify the read buffer size in bytes.
 The default is 4096 bytes.
 Attempts to set a buffer size lower than this will be silently
@@ -92,15 +124,43 @@ The number of reads actually performed is reported at verbosity level
 two or higher (see the
 .Fl v
 flag).
+.It Fl -bind-address= Ns Ar host
+Specifies a hostname or IP address to which sockets used for outgoing
+connections will be bound.
 .It Fl c Ar dir
 The file to retrieve is in directory
 .Ar dir
 on the remote host.
 This option is deprecated and is provided for backward compatibility
 only.
-.It Fl d
+.It Fl -ca-cert= Ns Ar file
+[SSL]
+Path to certificate bundle containing trusted CA certificates. 
+If not specified,
+.Pa /etc/ssl/cert.pem
+is used.
+The file may contain multiple CA certificates. The port
+.Pa security/ca_root_nss
+is a common source of a current CA bundle.
+.It Fl -ca-path= Ns Ar dir
+[SSL]
+The directory
+.Ar dir
+contains trusted CA hashes.
+.It Fl -cert= Ns Ar file
+[SSL]
+.Ar file
+is a PEM encoded client certificate/key which will be used in
+client certificate authentication.
+.It Fl -crl= Ns Ar file
+[SSL]
+Points to certificate revocation list
+.Ar file ,
+which has to be in PEM format and may contain peer certificates that have
+been revoked.
+.It Fl d , -direct
 Use a direct connection even if a proxy is configured.
-.It Fl F
+.It Fl F , -force-restart
 In combination with the
 .Fl r
 flag, forces a restart even if the local and remote files have
@@ -118,17 +178,22 @@ The file to retrieve is located on the host
 .Ar host .
 This option is deprecated and is provided for backward compatibility
 only.
-.It Fl i Ar file
+.It Fl i Ar file , Fl -if-modified-since= Ns Ar file
 If-Modified-Since mode: the remote file will only be retrieved if it
 is newer than
 .Ar file
 on the local host.
 (HTTP only)
-.It Fl l
+.It Fl -key= Ns Ar file
+[SSL]
+.Ar file
+is a PEM encoded client key that will be used in client certificate
+authentication in case key and client certificate are stored separately.
+.It Fl l , -symlink
 If the target is a file-scheme URL, make a symbolic link to the target
 rather than trying to copy it.
 .It Fl M
-.It Fl m
+.It Fl m , -mirror
 Mirror mode: if the file already exists locally and has the same size
 and modification time as the remote file, it will not be fetched.
 Note that the
@@ -136,7 +201,7 @@ Note that the
 and
 .Fl r
 flags are mutually exclusive.
-.It Fl N Ar file
+.It Fl N Ar file , Fl -netrc= Ns Ar file
 Use
 .Ar file
 instead of
@@ -146,9 +211,28 @@ See
 .Xr ftp 1
 for a description of the file format.
 This feature is experimental.
-.It Fl n
+.It Fl n , -no-mtime
 Do not preserve the modification time of the transferred file.
-.It Fl o Ar file
+.It Fl -no-passive
+Forces the FTP code to use active mode.
+.It Fl -no-proxy= Ns Ar list
+Either a single asterisk, which disables the use of proxies
+altogether, or a comma- or whitespace-separated list of hosts for
+which proxies should not be used.
+.It Fl -no-sslv3
+[SSL]
+Don't allow SSL version 3 when negotiating the connection.
+.It Fl -no-tlsv1
+[SSL]
+Don't allow TLS version 1 when negotiating the connection.
+.It Fl -no-verify-hostname
+[SSL]
+Do not verify that the hostname matches the subject of the
+certificate presented by the server.
+.It Fl -no-verify-peer
+[SSL]
+Do not verify the peer certificate against trusted CAs.
+.It Fl o Ar file , Fl output= Ns Ar file
 Set the output file name to
 .Ar file .
 By default, a ``pathname'' is extracted from the specified URI, and
@@ -163,36 +247,45 @@ If the
 argument is a directory, fetched file(s) will be placed within the
 directory, with name(s) selected as in the default behaviour.
 .It Fl P
-.It Fl p
+.It Fl p , -passive
 Use passive FTP.
 These flags have no effect, since passive FTP is the default, but are
 provided for compatibility with earlier versions where active FTP was
 the default.
-To force active mode, set the
+To force active mode, use the
+.Fl -no-passive
+flag or set the
 .Ev FTP_PASSIVE_MODE
 environment variable to
 .Ql NO .
-.It Fl q
+.It Fl -referer= Ns Ar URL
+Specifies the referrer URL to use for HTTP requests.
+If
+.Ar URL
+is set to
+.Dq auto ,
+the document URL will be used as referrer URL.
+.It Fl q , -quiet
 Quiet mode.
-.It Fl R
+.It Fl R , -keep-output
 The output files are precious, and should not be deleted under any
 circumstances, even if the transfer failed or was incomplete.
-.It Fl r
+.It Fl r , -restart
 Restart a previously interrupted transfer.
 Note that the
 .Fl m
 and
 .Fl r
 flags are mutually exclusive.
-.It Fl S Ar bytes
+.It Fl S Ar bytes , Fl -require-size= Ns Ar bytes
 Require the file size reported by the server to match the specified
 value.
 If it does not, a message is printed and the file is not fetched.
 If the server does not support reporting file sizes, this option is
 ignored and the file is fetched unconditionally.
-.It Fl s
+.It Fl s , -print-size
 Print the size in bytes of each requested file, without fetching it.
-.It Fl T Ar seconds
+.It Fl T Ar seconds , Fl -timeout= Ns Ar seconds
 Set timeout value to
 .Ar seconds .
 Overrides the environment variables
@@ -200,15 +293,19 @@ Overrides the environment variables
 for FTP transfers or
 .Ev HTTP_TIMEOUT
 for HTTP transfers if set.
-.It Fl U
+.It Fl U , -passive-portrange-default
 When using passive FTP, allocate the port for the data connection from
 the low (default) port range.
 See
 .Xr ip 4
 for details on how to specify which port range this corresponds to.
-.It Fl v
+.It Fl -user-agent= Ns Ar agent-string
+Specifies the User-Agent string to use for HTTP requests.
+This can be useful when working with HTTP origin or proxy servers that
+differentiate between user agents.
+.It Fl v , -verbose
 Increase verbosity level.
-.It Fl w Ar seconds
+.It Fl w Ar seconds , Fl -retry-delay= Ns Ar seconds
 When the
 .Fl a
 flag is specified, wait this many seconds between successive retries.
@@ -249,9 +346,19 @@ for a description of additional environment variables, including
 .Ev HTTP_REFERER ,
 .Ev HTTP_USER_AGENT ,
 .Ev NETRC ,
-.Ev NO_PROXY
+.Ev NO_PROXY ,
+.Ev no_proxy ,
+.Ev SSL_ALLOW_SSL2 ,
+.Ev SSL_CA_CERT_FILE ,
+.Ev SSL_CA_CERT_PATH ,
+.Ev SSL_CLIENT_CERT_FILE ,
+.Ev SSL_CLIENT_KEY_FILE ,
+.Ev SSL_CRL_FILE ,
+.Ev SSL_NO_SSL3 ,
+.Ev SSL_NO_TLS1 ,
+.Ev SSL_NO_VERIFY_HOSTNAME
 and
-.Ev no_proxy .
+.Ev SSL_NO_VERIFY_PEER .
 .Sh EXIT STATUS
 The
 .Nm
@@ -288,7 +395,9 @@ by
 and later completely rewritten to use the
 .Xr fetch 3
 library by
-.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org .
+.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org
+and
+.An Michael Gmelin Aq freebsd@grem.de .
 .Sh NOTES
 The
 .Fl b
diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c
index 27211b3..c61e88e 100644
--- a/usr.bin/fetch/fetch.c
+++ b/usr.bin/fetch/fetch.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2000-2011 Dag-Erling Smørgrav
+ * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
+#include <getopt.h>
 #include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -93,6 +95,78 @@ static long	 ftp_timeout = TIMEOUT;	/* default timeout for FTP transfers */
 static long	 http_timeout = TIMEOUT;/* default timeout for HTTP transfers */
 static char	*buf;		/* transfer buffer */
 
+enum options
+{
+	OPTION_BIND_ADDRESS,
+	OPTION_NO_FTP_PASSIVE_MODE,
+	OPTION_HTTP_REFERER,
+	OPTION_HTTP_USER_AGENT,
+	OPTION_NO_PROXY,
+	OPTION_SSL_ALLOW_SSL2,
+	OPTION_SSL_CA_CERT_FILE,
+	OPTION_SSL_CA_CERT_PATH,
+	OPTION_SSL_CLIENT_CERT_FILE,
+	OPTION_SSL_CLIENT_KEY_FILE,
+	OPTION_SSL_CRL_FILE,
+	OPTION_SSL_NO_SSL3,
+	OPTION_SSL_NO_TLS1,	
+	OPTION_SSL_NO_VERIFY_HOSTNAME,
+	OPTION_SSL_NO_VERIFY_PEER
+};
+
+
+static struct option longopts[] =
+{
+	/* mapping to single character argument */
+	{ "one-file", no_argument, NULL, '1' },
+	{ "ipv4-only", no_argument, NULL, '4' },
+	{ "ipv6-only", no_argument, NULL, '6' },
+	{ "no-redirect", no_argument, NULL, 'A' },
+	{ "retry", no_argument, NULL, 'a' },
+	{ "buffer-size", required_argument, NULL, 'B' },
+	/* -c not mapped, since it's deprecated */
+	{ "direct", no_argument, NULL, 'd' },
+	{ "force-restart", no_argument, NULL, 'F' },
+	/* -f not mapped, since it's deprecated */
+	/* -h not mapped, since it's deprecated */
+	{ "if-modified-since", required_argument, NULL, 'i' },
+	{ "symlink", no_argument, NULL, 'l' },
+	/* -M not mapped since it's the same as -m */
+	{ "mirror", no_argument, NULL, 'm' },
+	{ "netrc", required_argument, NULL, 'N' },
+	{ "no-mtime", no_argument, NULL, 'n' },
+	{ "output", required_argument, NULL, 'o' },
+	/* -P not mapped since it's the same as -p */
+	{ "passive", no_argument, NULL, 'p' },
+	{ "quiet", no_argument, NULL, 'q' },
+	{ "keep-output", no_argument, NULL, 'R' },
+	{ "restart", no_argument, NULL, 'r' },
+	{ "require-size", required_argument, NULL, 'S' },
+	{ "print-size", no_argument, NULL, 's' },
+	{ "timeout", required_argument, NULL, 'T' },
+	{ "passive-portrange-default", no_argument, NULL, 'T' },
+	{ "verbose", no_argument, NULL, 'v' },
+	{ "retry-delay", required_argument, NULL, 'w' },
+	
+	/* options without a single character equivalent */
+	{ "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS },
+	{ "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE },
+	{ "referer", required_argument, NULL, OPTION_HTTP_REFERER },
+	{ "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT },
+	{ "no-proxy", required_argument, NULL, OPTION_NO_PROXY },
+	{ "allow-sslv2", no_argument, NULL, OPTION_SSL_ALLOW_SSL2 },
+	{ "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE },
+	{ "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH },
+	{ "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE },
+	{ "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE },
+	{ "crl", required_argument, NULL, OPTION_SSL_CRL_FILE },
+	{ "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 },
+	{ "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 },
+	{ "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME },
+	{ "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER },
+
+	{ NULL, 0, NULL, 0 }
+};
 
 /*
  * Signal handler
@@ -769,11 +843,19 @@ fetch(char *URL, const char *path)
 static void
 usage(void)
 {
-	fprintf(stderr, "%s\n%s\n%s\n%s\n",
-"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
-"       [-T seconds] [-w seconds] [-i file] URL ...",
-"       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
-"       [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]");
+	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
+"usage: fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]",
+"       [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]",
+"       [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]",
+"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
+"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"       [--user-agent=agent-string] [-w seconds] URL ...",
+"       fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]",
+"       [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]",
+"       [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]",
+"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
+"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"       [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
 }
 
 
@@ -789,8 +871,10 @@ main(int argc, char *argv[])
 	char *end, *q;
 	int c, e, r;
 
-	while ((c = getopt(argc, argv,
-	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
+
+	while ((c = getopt_long(argc, argv,
+	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:",
+	    longopts, NULL)) != -1)
 		switch (c) {
 		case '1':
 			once_flag = 1;
@@ -904,6 +988,51 @@ main(int argc, char *argv[])
 			if (*optarg == '\0' || *end != '\0')
 				errx(1, "invalid delay (%s)", optarg);
 			break;
+		case OPTION_BIND_ADDRESS:
+			setenv("FETCH_BIND_ADDRESS", optarg, 1);
+			break;
+		case OPTION_NO_FTP_PASSIVE_MODE:
+			setenv("FTP_PASSIVE_MODE", "no", 1);
+			break;
+		case OPTION_HTTP_REFERER:
+			setenv("HTTP_REFERER", optarg, 1);
+			break;
+		case OPTION_HTTP_USER_AGENT:
+			setenv("HTTP_USER_AGENT", optarg, 1);
+			break;
+		case OPTION_NO_PROXY:
+			setenv("NO_PROXY", optarg, 1);
+			break;
+		case OPTION_SSL_ALLOW_SSL2:
+			setenv("SSL_ALLOW_SSL2", "", 1);
+			break;
+		case OPTION_SSL_CA_CERT_FILE:
+			setenv("SSL_CA_CERT_FILE", optarg, 1);
+			break;
+		case OPTION_SSL_CA_CERT_PATH:
+			setenv("SSL_CA_CERT_PATH", optarg, 1);
+			break;
+		case OPTION_SSL_CLIENT_CERT_FILE:
+			setenv("SSL_CLIENT_CERT_FILE", optarg, 1);
+			break;
+		case OPTION_SSL_CLIENT_KEY_FILE:
+			setenv("SSL_CLIENT_KEY_FILE", optarg, 1);
+			break;
+		case OPTION_SSL_CRL_FILE:
+			setenv("SSL_CLIENT_CRL_FILE", optarg, 1);
+			break;
+		case OPTION_SSL_NO_SSL3:
+			setenv("SSL_NO_SSL3", "", 1);
+			break;
+		case OPTION_SSL_NO_TLS1:
+			setenv("SSL_NO_TLS1", "", 1);
+			break;
+		case OPTION_SSL_NO_VERIFY_HOSTNAME:
+			setenv("SSL_NO_VERIFY_HOSTNAME", "", 1);
+			break;
+		case OPTION_SSL_NO_VERIFY_PEER:
+			setenv("SSL_NO_VERIFY_PEER", "", 1);
+			break;
 		default:
 			usage();
 			exit(1);