diff options
| -rw-r--r-- | lib/libfetch/common.c | 535 | ||||
| -rw-r--r-- | lib/libfetch/common.h | 5 | ||||
| -rw-r--r-- | lib/libfetch/fetch.3 | 106 | ||||
| -rw-r--r-- | lib/libfetch/http.c | 2 | ||||
| -rw-r--r-- | usr.bin/fetch/fetch.1 | 165 | ||||
| -rw-r--r-- | usr.bin/fetch/fetch.c | 143 |
6 files changed, 905 insertions, 51 deletions
diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c index 425b6de..15d5a25 100644 --- a/lib/libfetch/common.c +++ b/lib/libfetch/common.c @@ -1,5 +1,6 @@ /*- * Copyright (c) 1998-2011 Dag-Erling Smørgrav + * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -47,6 +48,10 @@ __FBSDID("$FreeBSD$"); #include <string.h> #include <unistd.h> +#ifdef WITH_SSL +#include <openssl/x509v3.h> +#endif + #include "fetch.h" #include "common.h" @@ -317,15 +322,488 @@ fetch_connect(const char *host, int port, int af, int verbose) return (conn); } +#ifdef WITH_SSL +/* + * Convert characters A-Z to lowercase (intentionally avoid any locale + * specific conversions). + */ +static char +fetch_ssl_tolower(char in) +{ + if (in >= 'A' && in <= 'Z') + return (in + 32); + else + return (in); +} + +/* + * isalpha implementation that intentionally avoids any locale specific + * conversions. + */ +static int +fetch_ssl_isalpha(char in) +{ + return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z')); +} + +/* + * Check if passed hostnames a and b are equal. + */ +static int +fetch_ssl_hname_equal(const char *a, size_t alen, const char *b, + size_t blen) +{ + size_t i; + + if (alen != blen) + return (0); + for (i = 0; i < alen; ++i) { + if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i])) + return (0); + } + return (1); +} + +/* + * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9 + * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha- + * numeric characters. Double hyphens (like they're found in IDN a-labels + * 'xn--') are not allowed. Empty labels are invalid. + */ +static int +fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok) +{ + size_t i; + + if (!len || l[0] == '-' || l[len-1] == '-') + return (0); + for (i = 0; i < len; ++i) { + if (!isdigit(l[i]) && + !fetch_ssl_isalpha(l[i]) && + !(l[i] == '*' && wcok) && + !(l[i] == '-' && l[i - 1] != '-')) + return (0); + } + return (1); +} + +/* + * Check if host name consists only of numbers. This might indicate an IP + * address, which is not a good idea for CN wildcard comparison. + */ +static int +fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len) +{ + size_t i; + + for (i = 0; i < len; ++i) { + if (!((hostname[i] >= '0' && hostname[i] <= '9') || + hostname[i] == '.')) + return (0); + } + return (1); +} + +/* + * Check if the host name h passed matches the pattern passed in m which + * is usually part of subjectAltName or CN of a certificate presented to + * the client. This includes wildcard matching. The algorithm is based on + * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280. + */ +static int +fetch_ssl_hname_match(const char *h, size_t hlen, const char *m, + size_t mlen) +{ + int delta, hdotidx, mdot1idx, wcidx; + const char *hdot, *mdot1, *mdot2; + const char *wc; /* wildcard */ + + if (!(h && *h && m && *m)) + return (0); + if ((wc = strnstr(m, "*", mlen)) == NULL) + return (fetch_ssl_hname_equal(h, hlen, m, mlen)); + wcidx = wc - m; + /* hostname should not be just dots and numbers */ + if (fetch_ssl_hname_is_only_numbers(h, hlen)) + return (0); + /* only one wildcard allowed in pattern */ + if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL) + return (0); + /* + * there must be at least two more domain labels and + * wildcard has to be in the leftmost label (RFC6125) + */ + mdot1 = strnstr(m, ".", mlen); + if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4) + return (0); + mdot1idx = mdot1 - m; + mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1); + if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2) + return (0); + /* hostname must contain a dot and not be the 1st char */ + hdot = strnstr(h, ".", hlen); + if (hdot == NULL || hdot == h) + return (0); + hdotidx = hdot - h; + /* + * host part of hostname must be at least as long as + * pattern it's supposed to match + */ + if (hdotidx < mdot1idx) + return (0); + /* + * don't allow wildcards in non-traditional domain names + * (IDN, A-label, U-label...) + */ + if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) || + !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1)) + return (0); + /* match domain part (part after first dot) */ + if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1, + mlen - mdot1idx)) + return (0); + /* match part left of wildcard */ + if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx)) + return (0); + /* match part right of wildcard */ + delta = mdot1idx - wcidx - 1; + if (!fetch_ssl_hname_equal(hdot - delta, delta, + mdot1 - delta, delta)) + return (0); + /* all tests succeded, it's a match */ + return (1); +} + +/* + * Get numeric host address info - returns NULL if host was not an IP + * address. The caller is responsible for deallocation using + * freeaddrinfo(3). + */ +static struct addrinfo * +fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len) +{ + struct addrinfo hints, *res; + char *host; + + host = (char *)malloc(len + 1); + memcpy(host, hostname, len); + host[len] = '\0'; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + hints.ai_flags = AI_NUMERICHOST; + /* port is not relevant for this purpose */ + getaddrinfo(host, "443", &hints, &res); + free(host); + return res; +} + +/* + * Compare ip address in addrinfo with address passes. + */ +static int +fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost, + size_t rhostlen) +{ + const void *left; + + if (lhost->ai_family == AF_INET && rhostlen == 4) { + left = (void *)&((struct sockaddr_in*)(void *) + lhost->ai_addr)->sin_addr.s_addr; +#ifdef INET6 + } else if (lhost->ai_family == AF_INET6 && rhostlen == 16) { + left = (void *)&((struct sockaddr_in6 *)(void *) + lhost->ai_addr)->sin6_addr; +#endif + } else + return (0); + return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0); +} + +/* + * Compare ip address in addrinfo with host passed. If host is not an IP + * address, comparison will fail. + */ +static int +fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r, + size_t rlen) +{ + struct addrinfo *raddr; + int ret; + char *rip; + + ret = 0; + if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL) + return 0; /* not a numeric host */ + + if (laddr->ai_family == raddr->ai_family) { + if (laddr->ai_family == AF_INET) { + rip = (char *)&((struct sockaddr_in *)(void *) + raddr->ai_addr)->sin_addr.s_addr; + ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4); +#ifdef INET6 + } else if (laddr->ai_family == AF_INET6) { + rip = (char *)&((struct sockaddr_in6 *)(void *) + raddr->ai_addr)->sin6_addr; + ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16); +#endif + } + + } + freeaddrinfo(raddr); + return (ret); +} + +/* + * Verify server certificate by subjectAltName. + */ +static int +fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames, + const char *host, struct addrinfo *ip) +{ + const GENERAL_NAME *name; + size_t nslen; + int i; + const char *ns; + + for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) { +#if OPENSSL_VERSION_NUMBER < 0x10000000L + /* + * This is a workaround, since the following line causes + * alignment issues in clang: + * name = sk_GENERAL_NAME_value(altnames, i); + * OpenSSL explicitly warns not to use those macros + * directly, but there isn't much choice (and there + * shouldn't be any ill side effects) + */ + name = (GENERAL_NAME *)SKM_sk_value(void, altnames, i); +#else + name = sk_GENERAL_NAME_value(altnames, i); +#endif + ns = (const char *)ASN1_STRING_data(name->d.ia5); + nslen = (size_t)ASN1_STRING_length(name->d.ia5); + + if (name->type == GEN_DNS && ip == NULL && + fetch_ssl_hname_match(host, strlen(host), ns, nslen)) + return (1); + else if (name->type == GEN_IPADD && ip != NULL && + fetch_ssl_ipaddr_match_bin(ip, ns, nslen)) + return (1); + } + return (0); +} + +/* + * Verify server certificate by CN. + */ +static int +fetch_ssl_verify_cn(X509_NAME *subject, const char *host, + struct addrinfo *ip) +{ + ASN1_STRING *namedata; + X509_NAME_ENTRY *nameentry; + int cnlen, lastpos, loc, ret; + unsigned char *cn; + + ret = 0; + lastpos = -1; + loc = -1; + cn = NULL; + /* get most specific CN (last entry in list) and compare */ + while ((lastpos = X509_NAME_get_index_by_NID(subject, + NID_commonName, lastpos)) != -1) + loc = lastpos; + + if (loc > -1) { + nameentry = X509_NAME_get_entry(subject, loc); + namedata = X509_NAME_ENTRY_get_data(nameentry); + cnlen = ASN1_STRING_to_UTF8(&cn, namedata); + if (ip == NULL && + fetch_ssl_hname_match(host, strlen(host), cn, cnlen)) + ret = 1; + else if (ip != NULL && fetch_ssl_ipaddr_match(ip, cn, cnlen)) + ret = 1; + OPENSSL_free(cn); + } + return (ret); +} + +/* + * Verify that server certificate subjectAltName/CN matches + * hostname. First check, if there are alternative subject names. If yes, + * those have to match. Only if those don't exist it falls back to + * checking the subject's CN. + */ +static int +fetch_ssl_verify_hname(X509 *cert, const char *host) +{ + struct addrinfo *ip; + STACK_OF(GENERAL_NAME) *altnames; + X509_NAME *subject; + int ret; + + ret = 0; + ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host)); + altnames = X509_get_ext_d2i(cert, NID_subject_alt_name, + NULL, NULL); + + if (altnames != NULL) { + ret = fetch_ssl_verify_altname(altnames, host, ip); + } else { + subject = X509_get_subject_name(cert); + if (subject != NULL) + ret = fetch_ssl_verify_cn(subject, host, ip); + } + + if (ip != NULL) + freeaddrinfo(ip); + if (altnames != NULL) + GENERAL_NAMES_free(altnames); + return (ret); +} + +/* + * Configure transport security layer based on environment. + */ +static void +fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose) +{ + long ssl_ctx_options; + + ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_TICKET; + if (getenv("SSL_ALLOW_SSL2") == NULL) + ssl_ctx_options |= SSL_OP_NO_SSLv2; + if (getenv("SSL_NO_SSL3") != NULL) + ssl_ctx_options |= SSL_OP_NO_SSLv3; + if (getenv("SSL_NO_TLS1") != NULL) + ssl_ctx_options |= SSL_OP_NO_TLSv1; + if (verbose) + fetch_info("SSL options: %x", ssl_ctx_options); + SSL_CTX_set_options(ctx, ssl_ctx_options); +} + + +/* + * Configure peer verification based on environment. + */ +static int +fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose) +{ + X509_LOOKUP *crl_lookup; + X509_STORE *crl_store; + const char *ca_cert_file, *ca_cert_path, *crl_file; + + if (getenv("SSL_NO_VERIFY_PEER") == NULL) { + ca_cert_file = getenv("SSL_CA_CERT_FILE") != NULL ? + getenv("SSL_CA_CERT_FILE") : "/etc/ssl/cert.pem"; + ca_cert_path = getenv("SSL_CA_CERT_PATH"); + if (verbose) { + fetch_info("Peer verification enabled"); + if (ca_cert_file != NULL) + fetch_info("Using CA cert file: %s", + ca_cert_file); + if (ca_cert_path != NULL) + fetch_info("Using CA cert path: %s", + ca_cert_path); + } + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, + fetch_ssl_cb_verify_crt); + SSL_CTX_load_verify_locations(ctx, ca_cert_file, + ca_cert_path); + if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) { + if (verbose) + fetch_info("Using CRL file: %s", crl_file); + crl_store = SSL_CTX_get_cert_store(ctx); + crl_lookup = X509_STORE_add_lookup(crl_store, + X509_LOOKUP_file()); + if (crl_lookup == NULL || + !X509_load_crl_file(crl_lookup, crl_file, + X509_FILETYPE_PEM)) { + fprintf(stderr, + "Could not load CRL file %s\n", + crl_file); + return (0); + } + X509_STORE_set_flags(crl_store, + X509_V_FLAG_CRL_CHECK | + X509_V_FLAG_CRL_CHECK_ALL); + } + } + return (1); +} + +/* + * Configure client certificate based on environment. + */ +static int +fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose) +{ + const char *client_cert_file, *client_key_file; + + if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) { + client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ? + getenv("SSL_CLIENT_KEY_FILE") : client_cert_file; + if (verbose) { + fetch_info("Using client cert file: %s", + client_cert_file); + fetch_info("Using client key file: %s", + client_key_file); + } + if (SSL_CTX_use_certificate_chain_file(ctx, + client_cert_file) != 1) { + fprintf(stderr, + "Could not load client certificate %s\n", + client_cert_file); + return (0); + } + if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file, + SSL_FILETYPE_PEM) != 1) { + fprintf(stderr, + "Could not load client key %s\n", + client_key_file); + return (0); + } + } + return (1); +} + +/* + * Callback for SSL certificate verification, this is called on server + * cert verification. It takes no decision, but informs the user in case + * verification failed. + */ +int +fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx) +{ + X509 *crt; + X509_NAME *name; + char *str; + + str = NULL; + if (!verified) { + if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL && + (name = X509_get_subject_name(crt)) != NULL) + str = X509_NAME_oneline(name, 0, 0); + fprintf(stderr, "Certificate verification failed for %s\n", + str != NULL ? str : "no relevant certificate"); + OPENSSL_free(str); + } + return (verified); +} + +#endif /* * Enable SSL on a connection. */ int -fetch_ssl(conn_t *conn, int verbose) +fetch_ssl(conn_t *conn, const struct url *URL, int verbose) { #ifdef WITH_SSL int ret, ssl_err; + X509_NAME *name; + char *str; /* Init the SSL library and context */ if (!SSL_library_init()){ @@ -339,8 +817,14 @@ fetch_ssl(conn_t *conn, int verbose) conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth); SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY); + fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose); + if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose)) + return (-1); + if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose)) + return (-1); + conn->ssl = SSL_new(conn->ssl_ctx); - if (conn->ssl == NULL){ + if (conn->ssl == NULL) { fprintf(stderr, "SSL context creation failed\n"); return (-1); } @@ -353,22 +837,35 @@ fetch_ssl(conn_t *conn, int verbose) return (-1); } } + conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); - if (verbose) { - X509_NAME *name; - char *str; + if (conn->ssl_cert == NULL) { + fprintf(stderr, "No server SSL certificate\n"); + return (-1); + } + + if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) { + if (verbose) + fetch_info("Verify hostname"); + if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) { + fprintf(stderr, + "SSL certificate subject doesn't match host %s\n", + URL->host); + return (-1); + } + } - fprintf(stderr, "SSL connection established using %s\n", + if (verbose) { + fetch_info("SSL connection established using %s", SSL_get_cipher(conn->ssl)); - conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); name = X509_get_subject_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); - printf("Certificate subject: %s\n", str); - free(str); + fetch_info("Certificate subject: %s", str); + OPENSSL_free(str); name = X509_get_issuer_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); - printf("Certificate issuer: %s\n", str); - free(str); + fetch_info("Certificate issuer: %s", str); + OPENSSL_free(str); } return (0); @@ -726,6 +1223,22 @@ fetch_close(conn_t *conn) if (--conn->ref > 0) return (0); +#ifdef WITH_SSL + if (conn->ssl) { + SSL_shutdown(conn->ssl); + SSL_set_connect_state(conn->ssl); + SSL_free(conn->ssl); + conn->ssl = NULL; + } + if (conn->ssl_ctx) { + SSL_CTX_free(conn->ssl_ctx); + conn->ssl_ctx = NULL; + } + if (conn->ssl_cert) { + X509_free(conn->ssl_cert); + conn->ssl_cert = NULL; + } +#endif ret = close(conn->sd); free(conn->cache.buf); free(conn->buf); diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h index fe591d3..1d543a6 100644 --- a/lib/libfetch/common.h +++ b/lib/libfetch/common.h @@ -87,7 +87,10 @@ int fetch_bind(int, int, const char *); conn_t *fetch_connect(const char *, int, int, int); conn_t *fetch_reopen(int); conn_t *fetch_ref(conn_t *); -int fetch_ssl(conn_t *, int); +#ifdef WITH_SSL +int fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*); +#endif +int fetch_ssl(conn_t *, const struct url *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); ssize_t fetch_write(conn_t *, const char *, size_t); diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3 index 30372f2..34ca445 100644 --- a/lib/libfetch/fetch.3 +++ b/lib/libfetch/fetch.3 @@ -1,5 +1,6 @@ .\"- .\" Copyright (c) 1998-2011 Dag-Erling Smørgrav +.\" Copyright (c) 2013 Michael Gmelin <freebsd@grem.de> .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -25,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd September 27, 2011 +.Dd January 25, 2013 .Dt FETCH 3 .Os .Sh NAME @@ -392,6 +393,60 @@ method in a manner consistent with the rest of the library, .Fn fetchPutHTTP is currently unimplemented. +.Sh HTTPS SCHEME +Based on HTTP SCHEME. +By default the peer is verified using the CA bundle located in +.Pa /etc/ssl/cert.pem . +The file may contain multiple CA certificates. +A common source of a current CA bundle is +.Pa \%security/ca_root_nss . +.Pp +The CA bundle used for peer verification can be changed by setting the +environment variables +.Ev SSL_CA_CERT_FILE +to point to a concatenated bundle of trusted certificates and +.Ev SSL_CA_CERT_PATH +to point to a directory containing hashes of trusted CAs (see +.Xr verify 1 ) . +.Pp +A certificate revocation list (CRL) can be used by setting the +environment variable +.Ev SSL_CRL_FILE +(see +.Xr crl 1 ) . +.Pp +Peer verification can be disabled by setting the environment variable +.Ev SSL_NO_VERIFY_PEER . +Note that this also disables CRL checking. +.Pp +By default the service identity is verified according to the rules +detailed in RFC6125 (also known as hostname verification). +This feature can be disabled by setting the environment variable +.Ev SSL_NO_VERIFY_HOSTNAME . +.Pp +Client certificate based authentication is supported. +The environment variable +.Ev SSL_CLIENT_CERT_FILE +should be set to point to a file containing key and client certificate +to be used in PEM format. In case the key is stored in a separate +file, the environment variable +.Ev SSL_CLIENT_KEY_FILE +can be set to point to the key in PEM format. +In case the key uses a password, the user will be prompted on standard +input (see +.Xr PEM 3 ) . +.Pp +By default +.Nm libfetch +allows SSLv3 and TLSv1 when negotiating the connecting with the remote +peer. +You can change this behavior by setting the environment variable +.Ev SSL_ALLOW_SSL2 +to allow SSLv2 (not recommended) and +.Ev SSL_NO_SSL3 +or +.Ev SSL_NO_TLS1 +to disable the respective methods. .Sh AUTHENTICATION Apart from setting the appropriate environment variables and specifying the user name and password in the URL or the @@ -579,6 +634,31 @@ which proxies should not be used. Same as .Ev NO_PROXY , for compatibility. +.It Ev SSL_ALLOW_SSL2 +Allow SSL version 2 when negotiating the connection (not recommended). +.It Ev SSL_CA_CERT_FILE +CA certificate bundle containing trusted CA certificates. +Default value: +.Pa /etc/ssl/cert.pem . +.It Ev SSL_CA_CERT_PATH +Path containing trusted CA hashes. +.It Ev SSL_CLIENT_CERT_FILE +PEM encoded client certificate/key which will be used in +client certificate authentication. +.It Ev SSL_CLIENT_KEY_FILE +PEM encoded client key in case key and client certificate +are stored separately. +.It Ev SSL_CRL_FILE +File containing certificate revocation list. +.It Ev SSL_NO_SSL3 +Don't allow SSL version 3 when negotiating the connection. +.It Ev SSL_NO_TLS1 +Don't allow TLV version 1 when negotiating the connection. +.It Ev SSL_NO_VERIFY_HOSTNAME +If set, do not verify that the hostname matches the subject of the +certificate presented by the server. +.It Ev SSL_NO_VERIFY_PEER +If set, do not verify the peer certificate against trusted CAs. .El .Sh EXAMPLES To access a proxy server on @@ -610,6 +690,19 @@ as follows: .Bd -literal -offset indent NO_PROXY=localhost,127.0.0.1 .Ed +.Pp +Access HTTPS website without any certificate verification whatsoever: +.Bd -literal -offset indent +SSL_NO_VERIFY_PEER=1 +SSL_NO_VERIFY_HOSTNAME=1 +.Ed +.Pp +Access HTTPS website using client certificate based authentication +and a private CA: +.Bd -literal -offset indent +SSL_CLIENT_CERT_FILE=/path/to/client.pem +SSL_CA_CERT_FILE=/path/to/myca.pem +.Ed .Sh SEE ALSO .Xr fetch 1 , .Xr ftpio 3 , @@ -678,7 +771,8 @@ with numerous suggestions and contributions from .An Hajimu Umemoto Aq ume@FreeBSD.org , .An Henry Whincup Aq henry@techiebod.com , .An Jukka A. Ukkonen Aq jau@iki.fi , -.An Jean-Fran\(,cois Dockes Aq jf@dockes.org +.An Jean-Fran\(,cois Dockes Aq jf@dockes.org , +.An Michael Gmelin Aq freebsd@grem.de and others. It replaces the older .Nm ftpio @@ -688,7 +782,9 @@ and .An Jordan K. Hubbard Aq jkh@FreeBSD.org . .Pp This manual page was written by -.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org . +.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org +and +.An Michael Gmelin Aq freebsd@grem.de . .Sh BUGS Some parts of the library are not yet implemented. The most notable @@ -717,6 +813,10 @@ implemented, superfluous at this site" in an FTP context and .Fn fetchStatFTP does not check that the result of an MDTM command is a valid date. .Pp +In case password protected keys are used for client certificate based +authentication the user is prompted for the password on each and every +fetch operation. +.Pp The man page is incomplete, poorly written and produces badly formatted text. .Pp diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c index 4811fe0..1fbc5ff 100644 --- a/lib/libfetch/http.c +++ b/lib/libfetch/http.c @@ -1408,7 +1408,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags) http_get_reply(conn); } if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && - fetch_ssl(conn, verbose) == -1) { + fetch_ssl(conn, URL, verbose) == -1) { fetch_close(conn); /* grrr */ errno = EAUTH; diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1 index cea6f1b..c340649 100644 --- a/usr.bin/fetch/fetch.1 +++ b/usr.bin/fetch/fetch.1 @@ -38,22 +38,51 @@ .Sh SYNOPSIS .Nm .Op Fl 146AadFlMmnPpqRrsUv +.Op Fl -allow-sslv2 .Op Fl B Ar bytes +.Op Fl -bind-address= Ns Ar host +.Op Fl -ca-cert= Ns Ar file +.Op Fl -ca-path= Ns Ar dir +.Op Fl -cert= Ns Ar file +.Op Fl -crl= Ns Ar file .Op Fl i Ar file +.Op Fl -key= Ns Ar file .Op Fl N Ar file +.Op Fl -no-passive +.Op Fl -no-proxy= Ns Ar list +.Op Fl -no-sslv3 +.Op Fl -no-tlsv1 +.Op Fl -no-verify-hostname +.Op Fl -no-verify-peer .Op Fl o Ar file +.Op Fl -referer= Ns Ar URL .Op Fl S Ar bytes .Op Fl T Ar seconds +.Op Fl -user-agent= Ns Ar agent-string .Op Fl w Ar seconds .Ar URL ... .Nm .Op Fl 146AadFlMmnPpqRrsUv .Op Fl B Ar bytes +.Op Fl -bind-address= Ns Ar host +.Op Fl -ca-cert= Ns Ar file +.Op Fl -ca-path= Ns Ar dir +.Op Fl -cert= Ns Ar file +.Op Fl -crl= Ns Ar file .Op Fl i Ar file +.Op Fl -key= Ns Ar file .Op Fl N Ar file +.Op Fl -no-passive +.Op Fl -no-proxy= Ns Ar list +.Op Fl -no-sslv3 +.Op Fl -no-tlsv1 +.Op Fl -no-verify-hostname +.Op Fl -no-verify-peer .Op Fl o Ar file +.Op Fl -referer= Ns Ar URL .Op Fl S Ar bytes .Op Fl T Ar seconds +.Op Fl -user-agent= Ns Ar agent-string .Op Fl w Ar seconds .Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc .Sh DESCRIPTION @@ -67,23 +96,26 @@ command line. .Pp The following options are available: .Bl -tag -width Fl -.It Fl 1 +.It Fl 1 , -one-file Stop and return exit code 0 at the first successfully retrieved file. -.It Fl 4 +.It Fl 4 , -ipv4-only Forces .Nm to use IPv4 addresses only. -.It Fl 6 +.It Fl 6 , -ipv6-only Forces .Nm to use IPv6 addresses only. -.It Fl A +.It Fl A , -no-redirect Do not automatically follow ``temporary'' (302) redirects. Some broken Web sites will return a redirect instead of a not-found error when the requested object does not exist. -.It Fl a +.It Fl a , -retry Automatically retry the transfer upon soft failures. -.It Fl B Ar bytes +.It Fl -allow-sslv2 +[SSL] +Allow SSL version 2 when negotiating the connection. +.It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes Specify the read buffer size in bytes. The default is 4096 bytes. Attempts to set a buffer size lower than this will be silently @@ -92,15 +124,43 @@ The number of reads actually performed is reported at verbosity level two or higher (see the .Fl v flag). +.It Fl -bind-address= Ns Ar host +Specifies a hostname or IP address to which sockets used for outgoing +connections will be bound. .It Fl c Ar dir The file to retrieve is in directory .Ar dir on the remote host. This option is deprecated and is provided for backward compatibility only. -.It Fl d +.It Fl -ca-cert= Ns Ar file +[SSL] +Path to certificate bundle containing trusted CA certificates. +If not specified, +.Pa /etc/ssl/cert.pem +is used. +The file may contain multiple CA certificates. The port +.Pa security/ca_root_nss +is a common source of a current CA bundle. +.It Fl -ca-path= Ns Ar dir +[SSL] +The directory +.Ar dir +contains trusted CA hashes. +.It Fl -cert= Ns Ar file +[SSL] +.Ar file +is a PEM encoded client certificate/key which will be used in +client certificate authentication. +.It Fl -crl= Ns Ar file +[SSL] +Points to certificate revocation list +.Ar file , +which has to be in PEM format and may contain peer certificates that have +been revoked. +.It Fl d , -direct Use a direct connection even if a proxy is configured. -.It Fl F +.It Fl F , -force-restart In combination with the .Fl r flag, forces a restart even if the local and remote files have @@ -118,17 +178,22 @@ The file to retrieve is located on the host .Ar host . This option is deprecated and is provided for backward compatibility only. -.It Fl i Ar file +.It Fl i Ar file , Fl -if-modified-since= Ns Ar file If-Modified-Since mode: the remote file will only be retrieved if it is newer than .Ar file on the local host. (HTTP only) -.It Fl l +.It Fl -key= Ns Ar file +[SSL] +.Ar file +is a PEM encoded client key that will be used in client certificate +authentication in case key and client certificate are stored separately. +.It Fl l , -symlink If the target is a file-scheme URL, make a symbolic link to the target rather than trying to copy it. .It Fl M -.It Fl m +.It Fl m , -mirror Mirror mode: if the file already exists locally and has the same size and modification time as the remote file, it will not be fetched. Note that the @@ -136,7 +201,7 @@ Note that the and .Fl r flags are mutually exclusive. -.It Fl N Ar file +.It Fl N Ar file , Fl -netrc= Ns Ar file Use .Ar file instead of @@ -146,9 +211,28 @@ See .Xr ftp 1 for a description of the file format. This feature is experimental. -.It Fl n +.It Fl n , -no-mtime Do not preserve the modification time of the transferred file. -.It Fl o Ar file +.It Fl -no-passive +Forces the FTP code to use active mode. +.It Fl -no-proxy= Ns Ar list +Either a single asterisk, which disables the use of proxies +altogether, or a comma- or whitespace-separated list of hosts for +which proxies should not be used. +.It Fl -no-sslv3 +[SSL] +Don't allow SSL version 3 when negotiating the connection. +.It Fl -no-tlsv1 +[SSL] +Don't allow TLS version 1 when negotiating the connection. +.It Fl -no-verify-hostname +[SSL] +Do not verify that the hostname matches the subject of the +certificate presented by the server. +.It Fl -no-verify-peer +[SSL] +Do not verify the peer certificate against trusted CAs. +.It Fl o Ar file , Fl output= Ns Ar file Set the output file name to .Ar file . By default, a ``pathname'' is extracted from the specified URI, and @@ -163,36 +247,45 @@ If the argument is a directory, fetched file(s) will be placed within the directory, with name(s) selected as in the default behaviour. .It Fl P -.It Fl p +.It Fl p , -passive Use passive FTP. These flags have no effect, since passive FTP is the default, but are provided for compatibility with earlier versions where active FTP was the default. -To force active mode, set the +To force active mode, use the +.Fl -no-passive +flag or set the .Ev FTP_PASSIVE_MODE environment variable to .Ql NO . -.It Fl q +.It Fl -referer= Ns Ar URL +Specifies the referrer URL to use for HTTP requests. +If +.Ar URL +is set to +.Dq auto , +the document URL will be used as referrer URL. +.It Fl q , -quiet Quiet mode. -.It Fl R +.It Fl R , -keep-output The output files are precious, and should not be deleted under any circumstances, even if the transfer failed or was incomplete. -.It Fl r +.It Fl r , -restart Restart a previously interrupted transfer. Note that the .Fl m and .Fl r flags are mutually exclusive. -.It Fl S Ar bytes +.It Fl S Ar bytes , Fl -require-size= Ns Ar bytes Require the file size reported by the server to match the specified value. If it does not, a message is printed and the file is not fetched. If the server does not support reporting file sizes, this option is ignored and the file is fetched unconditionally. -.It Fl s +.It Fl s , -print-size Print the size in bytes of each requested file, without fetching it. -.It Fl T Ar seconds +.It Fl T Ar seconds , Fl -timeout= Ns Ar seconds Set timeout value to .Ar seconds . Overrides the environment variables @@ -200,15 +293,19 @@ Overrides the environment variables for FTP transfers or .Ev HTTP_TIMEOUT for HTTP transfers if set. -.It Fl U +.It Fl U , -passive-portrange-default When using passive FTP, allocate the port for the data connection from the low (default) port range. See .Xr ip 4 for details on how to specify which port range this corresponds to. -.It Fl v +.It Fl -user-agent= Ns Ar agent-string +Specifies the User-Agent string to use for HTTP requests. +This can be useful when working with HTTP origin or proxy servers that +differentiate between user agents. +.It Fl v , -verbose Increase verbosity level. -.It Fl w Ar seconds +.It Fl w Ar seconds , Fl -retry-delay= Ns Ar seconds When the .Fl a flag is specified, wait this many seconds between successive retries. @@ -249,9 +346,19 @@ for a description of additional environment variables, including .Ev HTTP_REFERER , .Ev HTTP_USER_AGENT , .Ev NETRC , -.Ev NO_PROXY +.Ev NO_PROXY , +.Ev no_proxy , +.Ev SSL_ALLOW_SSL2 , +.Ev SSL_CA_CERT_FILE , +.Ev SSL_CA_CERT_PATH , +.Ev SSL_CLIENT_CERT_FILE , +.Ev SSL_CLIENT_KEY_FILE , +.Ev SSL_CRL_FILE , +.Ev SSL_NO_SSL3 , +.Ev SSL_NO_TLS1 , +.Ev SSL_NO_VERIFY_HOSTNAME and -.Ev no_proxy . +.Ev SSL_NO_VERIFY_PEER . .Sh EXIT STATUS The .Nm @@ -288,7 +395,9 @@ by and later completely rewritten to use the .Xr fetch 3 library by -.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org . +.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org +and +.An Michael Gmelin Aq freebsd@grem.de . .Sh NOTES The .Fl b diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c index 27211b3..c61e88e 100644 --- a/usr.bin/fetch/fetch.c +++ b/usr.bin/fetch/fetch.c @@ -1,5 +1,6 @@ /*- * Copyright (c) 2000-2011 Dag-Erling Smørgrav + * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include <ctype.h> #include <err.h> #include <errno.h> +#include <getopt.h> #include <signal.h> #include <stdint.h> #include <stdio.h> @@ -93,6 +95,78 @@ static long ftp_timeout = TIMEOUT; /* default timeout for FTP transfers */ static long http_timeout = TIMEOUT;/* default timeout for HTTP transfers */ static char *buf; /* transfer buffer */ +enum options +{ + OPTION_BIND_ADDRESS, + OPTION_NO_FTP_PASSIVE_MODE, + OPTION_HTTP_REFERER, + OPTION_HTTP_USER_AGENT, + OPTION_NO_PROXY, + OPTION_SSL_ALLOW_SSL2, + OPTION_SSL_CA_CERT_FILE, + OPTION_SSL_CA_CERT_PATH, + OPTION_SSL_CLIENT_CERT_FILE, + OPTION_SSL_CLIENT_KEY_FILE, + OPTION_SSL_CRL_FILE, + OPTION_SSL_NO_SSL3, + OPTION_SSL_NO_TLS1, + OPTION_SSL_NO_VERIFY_HOSTNAME, + OPTION_SSL_NO_VERIFY_PEER +}; + + +static struct option longopts[] = +{ + /* mapping to single character argument */ + { "one-file", no_argument, NULL, '1' }, + { "ipv4-only", no_argument, NULL, '4' }, + { "ipv6-only", no_argument, NULL, '6' }, + { "no-redirect", no_argument, NULL, 'A' }, + { "retry", no_argument, NULL, 'a' }, + { "buffer-size", required_argument, NULL, 'B' }, + /* -c not mapped, since it's deprecated */ + { "direct", no_argument, NULL, 'd' }, + { "force-restart", no_argument, NULL, 'F' }, + /* -f not mapped, since it's deprecated */ + /* -h not mapped, since it's deprecated */ + { "if-modified-since", required_argument, NULL, 'i' }, + { "symlink", no_argument, NULL, 'l' }, + /* -M not mapped since it's the same as -m */ + { "mirror", no_argument, NULL, 'm' }, + { "netrc", required_argument, NULL, 'N' }, + { "no-mtime", no_argument, NULL, 'n' }, + { "output", required_argument, NULL, 'o' }, + /* -P not mapped since it's the same as -p */ + { "passive", no_argument, NULL, 'p' }, + { "quiet", no_argument, NULL, 'q' }, + { "keep-output", no_argument, NULL, 'R' }, + { "restart", no_argument, NULL, 'r' }, + { "require-size", required_argument, NULL, 'S' }, + { "print-size", no_argument, NULL, 's' }, + { "timeout", required_argument, NULL, 'T' }, + { "passive-portrange-default", no_argument, NULL, 'T' }, + { "verbose", no_argument, NULL, 'v' }, + { "retry-delay", required_argument, NULL, 'w' }, + + /* options without a single character equivalent */ + { "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS }, + { "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE }, + { "referer", required_argument, NULL, OPTION_HTTP_REFERER }, + { "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT }, + { "no-proxy", required_argument, NULL, OPTION_NO_PROXY }, + { "allow-sslv2", no_argument, NULL, OPTION_SSL_ALLOW_SSL2 }, + { "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE }, + { "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH }, + { "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE }, + { "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE }, + { "crl", required_argument, NULL, OPTION_SSL_CRL_FILE }, + { "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 }, + { "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 }, + { "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME }, + { "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER }, + + { NULL, 0, NULL, 0 } +}; /* * Signal handler @@ -769,11 +843,19 @@ fetch(char *URL, const char *path) static void usage(void) { - fprintf(stderr, "%s\n%s\n%s\n%s\n", -"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", -" [-T seconds] [-w seconds] [-i file] URL ...", -" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", -" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]"); + fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", +"usage: fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]", +" [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]", +" [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]", +" [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]", +" [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]", +" [--user-agent=agent-string] [-w seconds] URL ...", +" fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]", +" [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]", +" [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]", +" [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]", +" [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]", +" [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]"); } @@ -789,8 +871,10 @@ main(int argc, char *argv[]) char *end, *q; int c, e, r; - while ((c = getopt(argc, argv, - "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) + + while ((c = getopt_long(argc, argv, + "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:", + longopts, NULL)) != -1) switch (c) { case '1': once_flag = 1; @@ -904,6 +988,51 @@ main(int argc, char *argv[]) if (*optarg == '\0' || *end != '\0') errx(1, "invalid delay (%s)", optarg); break; + case OPTION_BIND_ADDRESS: + setenv("FETCH_BIND_ADDRESS", optarg, 1); + break; + case OPTION_NO_FTP_PASSIVE_MODE: + setenv("FTP_PASSIVE_MODE", "no", 1); + break; + case OPTION_HTTP_REFERER: + setenv("HTTP_REFERER", optarg, 1); + break; + case OPTION_HTTP_USER_AGENT: + setenv("HTTP_USER_AGENT", optarg, 1); + break; + case OPTION_NO_PROXY: + setenv("NO_PROXY", optarg, 1); + break; + case OPTION_SSL_ALLOW_SSL2: + setenv("SSL_ALLOW_SSL2", "", 1); + break; + case OPTION_SSL_CA_CERT_FILE: + setenv("SSL_CA_CERT_FILE", optarg, 1); + break; + case OPTION_SSL_CA_CERT_PATH: + setenv("SSL_CA_CERT_PATH", optarg, 1); + break; + case OPTION_SSL_CLIENT_CERT_FILE: + setenv("SSL_CLIENT_CERT_FILE", optarg, 1); + break; + case OPTION_SSL_CLIENT_KEY_FILE: + setenv("SSL_CLIENT_KEY_FILE", optarg, 1); + break; + case OPTION_SSL_CRL_FILE: + setenv("SSL_CLIENT_CRL_FILE", optarg, 1); + break; + case OPTION_SSL_NO_SSL3: + setenv("SSL_NO_SSL3", "", 1); + break; + case OPTION_SSL_NO_TLS1: + setenv("SSL_NO_TLS1", "", 1); + break; + case OPTION_SSL_NO_VERIFY_HOSTNAME: + setenv("SSL_NO_VERIFY_HOSTNAME", "", 1); + break; + case OPTION_SSL_NO_VERIFY_PEER: + setenv("SSL_NO_VERIFY_PEER", "", 1); + break; default: usage(); exit(1); |
